rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Change the interface for parity allocation

Browse source 
This will allow a more compact representation
of the parity allocation
This commit is contained in:
Andrea Mazzoleni 2015-10-01 18:19:48 +02:00
parent 2801634aa4
commit 0407c945db
15 changed files with 449 additions and 286 deletions
Download patch file Download diff file Expand all files Collapse all files

12
TODO
View file

@ -56,6 +56,18 @@ Very fast to load from disk as we insert only at the end.
+ We can also get rid of deleted blocks, as the hash will be stored anyway.
+ We can put the state into the hash, dropping 3 bits of the hash of the 128 ones.
- A problem is that we are going to move hash from file to parity.
How to keep the hash of the file when the file is not in the parity ?
We'll have to allocate hash also in the file, and move data.
We can keep the hash ALWAYS on the file!
But how to handle DELETED blocks ? At now we handle them one-by-one,
We have to change to handle them like files, to allow to have DELETED files, inside the tree.
Here the problem is how to handle when DELETED blocks are overwritten.
Note that we need the UNDELETE state to allow to recover in FIX.
The HASH is also used to search for DELETED blocks.
TODO CONVERT DELETED BLOCKS TO DELETE FILES
* Restore ownership and permissions, at least in Unix.
* Restore directory timestamps.

6
cmdline/check.c
View file

@ -619,7 +619,7 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
continue;
}
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* if no file, nothing to do */
continue;
@ -819,7 +819,7 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct
if (!handle[j].disk)
continue;
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
/* try to recover all files, even the ones without hash */
/* because in some cases we can recover also them */
@ -943,7 +943,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
}
/* if the disk block is not used */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (block == BLOCK_EMPTY) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);

2
cmdline/dry.c
View file

@ -67,7 +67,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
continue;
}
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* if no file, nothing to do */

6
cmdline/dup.c
View file

@ -50,9 +50,11 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE);
struct snapraid_block* block = fs_file2block_get(disk, file, i);
if (!block_has_updated_hash(&file->blockvec[i])) {
memcpy(buf + i * HASH_SIZE, block->hash, HASH_SIZE);
if (!block_has_updated_hash(block)) {
free(buf);
free(hash);
return 0;

106
cmdline/elem.c
View file

@ -309,7 +309,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub)
return 1;
}
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file)
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file)
{
unsigned i;
@ -318,12 +318,8 @@ int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapr
/* check each block of the file */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block;
snapraid_info info;
block = &file->blockvec[i];
info = info_get(infoarr, block->parity_pos);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
snapraid_info info = info_get(infoarr, parity_pos);
/* if the file has a bad block, don't exclude it */
if (info_get_bad(info))
@ -444,7 +440,6 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
file->flag = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].parity_pos = POS_INVALID;
file->blockvec[i].file_mixed = 0;
@ -460,10 +455,40 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
return file;
}
struct snapraid_file* file_dup(struct snapraid_file* copy)
{
struct snapraid_file* file;
block_off_t i;
file = malloc_nofail(sizeof(struct snapraid_file));
file->sub = strdup_nofail(copy->sub);
file->size = copy->size;
file->blockmax = copy->blockmax;
file->mtime_sec = copy->mtime_sec;
file->mtime_nsec = copy->mtime_nsec;
file->inode = copy->inode;
file->physical = copy->physical;
file->flag = copy->flag;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].file_mixed = copy->blockvec[i].file_mixed;
file->blockvec[i].parity_pos = copy->blockvec[i].parity_pos;
memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE);
/* set the file to overwrite the copied one */
block_file_set(&file->blockvec[i], file);
}
return file;
}
void file_free(struct snapraid_file* file)
{
free(file->sub);
file->sub = 0;
free(file->blockvec);
file->blockvec = 0;
free(file);
}
@ -739,6 +764,71 @@ void disk_free(struct snapraid_disk* disk)
free(disk);
}
block_off_t disk_block_size(struct snapraid_disk* disk)
{
return tommy_arrayblk_size(&disk->blockarr);
}
void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos)
{
(void)disk;
assert(file_pos < file->blockmax);
file->blockvec[file_pos].parity_pos = parity_pos;
}
struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
{
struct snapraid_block* block = fs_par2block_get(disk, parity_pos);
if (block == BLOCK_EMPTY)
return 0;
if (file_pos)
*file_pos = block_file_pos(block);
return block_file_get(block);
}
block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
(void)disk;
assert(file_pos < file->blockmax);
return file->blockvec[file_pos].parity_pos;
}
struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
(void)disk;
assert(file_pos < file->blockmax);
return &file->blockvec[file_pos];
}
void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos)
{
tommy_arrayblk_grow(&disk->blockarr, pos + 1);
tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY);
}
void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block)
{
tommy_arrayblk_grow(&disk->blockarr, pos + 1);
tommy_arrayblk_set(&disk->blockarr, pos, block);
}
struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
{
if (parity_pos < tommy_arrayblk_size(&disk->blockarr))
return tommy_arrayblk_get(&disk->blockarr, parity_pos);
else
return BLOCK_EMPTY;
}
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax)
{
block_off_t i;

62
cmdline/elem.h
View file

@ -465,7 +465,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub);
* Filter a file if bad.
* Return !=0 if the file is correct and it should be excluded.
*/
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file);
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
/**
* Filters a dir using a list of filters.
@ -503,7 +503,7 @@ static inline void block_file_set(struct snapraid_block* block, struct snapraid_
/* LCOV_EXCL_STOP */
}
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | ptr;
block->file_mixed = (block->file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr;
}
/**
@ -700,12 +700,17 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
}
/**
* Allocates a file.
* Allocate a file.
*/
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
/**
* Deallocates a file.
* Duplicate a file.
*/
struct snapraid_file* file_dup(struct snapraid_file* copy);
/**
* Deallocate a file.
*/
void file_free(struct snapraid_file* file);
@ -884,16 +889,47 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
void disk_free(struct snapraid_disk* disk);
/**
* Gets a specific block of a disk.
* Returns 0 if the block is over the end of the disk or not used.
* Get the size of the disk in blocks.
*/
static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, block_off_t pos)
{
if (pos < tommy_arrayblk_size(&disk->blockarr))
return tommy_arrayblk_get(&disk->blockarr, pos);
else
return BLOCK_EMPTY;
}
block_off_t disk_block_size(struct snapraid_disk* disk);
/**
* Set a filesystem mapping between file and parity positions.
*/
void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
/**
* Get the file position from the parity position.
* Return 0 if no file is using it.
*/
struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
/**
* Get the parity position from the file position.
*/
block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
/**
* Get the block from the file position.
*/
struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
/**
* Clear a specific block.
*/
void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos);
/**
* Set the block from parity position.
* In case of holes, everything is initialized to 0.
*/
void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block);
/**
* Get the block from the parity position.
* Returns BLOCK_EMPTY==0 if the block is over the end of the disk or not used.
*/
struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos);
/**
* Check if a disk is totally empty and can be discarded from the content file.

6
cmdline/import.c
View file

@ -66,7 +66,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
file->path = strdup_nofail(path);
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
buffer = malloc_nofail(block_size);
@ -98,7 +98,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
offset = 0;
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_import_block* block = &file->blockvec[i];
struct snapraid_import_block* block = &file->blockimp[i];
unsigned read_size = block_size;
if (read_size > size)
read_size = size;
@ -145,7 +145,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
void import_file_free(struct snapraid_import_file* file)
{
free(file->path);
free(file->blockvec);
free(file->blockimp);
free(file);
}

2
cmdline/import.h
View file

@ -46,7 +46,7 @@ struct snapraid_import_block {
*/
struct snapraid_import_file {
data_off_t size; /**< Size of the file. */
struct snapraid_import_block* blockvec; /**< All the blocks of the file. */
struct snapraid_import_block* blockimp; /**< All the blocks of the file. */
block_off_t blockmax; /**< Number of blocks. */
char* path; /**< Full path of the file. */

13
cmdline/parity.c
View file

@ -37,12 +37,12 @@ block_off_t parity_allocated_size(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
block_off_t block = disk_block_size(disk);
/* decrease the block until an allocated one, but part of a file */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file(fs_par2block_get(disk, block - 1)))
--block;
/* get the highest value */
@ -64,10 +64,10 @@ block_off_t parity_used_size(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
block_off_t block = disk_block_size(disk);
/* decrease the block until an used one */
while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file_and_valid_parity(fs_par2block_get(disk, block - 1)))
--block;
/* get the highest value */
@ -95,7 +95,7 @@ int parity_is_invalid(struct snapraid_state* state)
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
struct snapraid_block* block = fs_par2block_get(disk, i);
if (block_has_file(block))
one_valid = 1;
@ -134,7 +134,8 @@ void parity_overflow(struct snapraid_state* state, data_off_t size)
struct snapraid_file* file = j->data;
if (file->blockmax > 0) {
if (file->blockvec[file->blockmax - 1].parity_pos >= blockalloc) {
block_off_t parity_pos = fs_file2par_get(disk, file, file->blockmax - 1);
if (parity_pos >= blockalloc) {
log_tag("outofparity:%s:%s\n", disk->name, esc(file->sub));
if (first) {
first = 0;

428
cmdline/scan.c
View file

@ -155,22 +155,116 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
}
/**
* Removes the specified file from the parity.
* Insert the specified file in the parity.
*/
static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t parity_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
parity_pos = disk->first_free_block;
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
/* increment the position until the first really free block */
while (block_has_file(fs_par2block_get(disk, parity_pos)))
++parity_pos;
/* set the position */
block->parity_pos = parity_pos;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(block, BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
struct snapraid_block* over_block;
unsigned block_state;
/* convert to a CHG block */
block_state_set(block, BLOCK_STATE_CHG);
/* block to overwrite */
over_block = fs_par2block_get(disk, parity_pos);
/* state of the block we are going to overwrite */
block_state = block_state_get(over_block);
/* if the block is an empty one */
if (block_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the past hash */
if (!state->clear_past_hash) {
/* in this case we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
fs_par2block_set(disk, parity_pos, block);
/* set the new free position */
disk->first_free_block = parity_pos + 1;
}
/* insert in the list of contained files */
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
/**
* Remove the specified file from the parity.
* The file is then inserted in the deleted set, and it should no be deallocated.
*/
static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
/* remove from the list of contained files */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* state changed */
state->need_write = 1;
/* free all the blocks of the file */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
block_off_t block_pos = block->parity_pos;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
unsigned block_state;
struct snapraid_deleted* deleted;
@ -178,8 +272,8 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
/* note that doing all the deletions before alllocations, */
/* first_free_block is always 0 and the "if" is never triggered */
/* but we keep this code anyway for completeness. */
if (disk->first_free_block > block_pos)
disk->first_free_block = block_pos;
if (disk->first_free_block > parity_pos)
disk->first_free_block = parity_pos;
/* allocated a new deleted block from the block we are going to delete */
deleted = deleted_dup(block);
@ -223,75 +317,13 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* set the deleted block in the block array */
tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block);
}
}
/**
* Checks if a file is completely formed of blocks with invalid parity,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has an invalid parity */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* exclude blocks with parity */
if (!block_has_invalid_parity(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* if rehash fails */
if (info_get_rehash(info))
return 0;
fs_par2block_set(disk, parity_pos, &deleted->block);
}
return 1;
file_free(file);
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has a hash */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* exclude blocks without hash */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* exclude blocks needing a rehash */
if (info_get_rehash(info))
return 0;
}
return 1;
}
static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file)
static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
@ -312,9 +344,76 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid
}
}
/* insert in the delayed list */
tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
}
/**
* Checks if a file is completely formed of blocks with invalid parity,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has an invalid parity */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
/* exclude blocks with parity */
if (!block_has_invalid_parity(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* if rehash fails */
if (info_get_rehash(info))
return 0;
}
return 1;
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has a hash */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
/* exclude blocks without hash */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* exclude blocks needing a rehash */
if (info_get_rehash(info))
return 0;
}
return 1;
}
/**
* Refresh the file info.
*
@ -399,7 +498,43 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
}
/**
* Keeps the file as it's (or with only a name/inode modification).
* Insert the file in the data set.
*/
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* insert the file in the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* delayed allocationg of the parity */
scan_file_delayed_allocate(scan, file);
}
/**
* Remove the file from the data set.
*
* File is then deleted.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* remove the file from the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
}
/**
* Keep the file as it's (or with only a name/inode modification).
*
* If the file is kept, nothing has to be done.
*
@ -414,139 +549,17 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* fil
struct snapraid_disk* disk = scan->disk;
/* if the file is full invalid, schedule a reinsert at later stage */
if (file_is_full_invalid_parity_and_stable(scan->state, file)) {
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
struct snapraid_file* copy = file_dup(file);
/* remove the file from the list */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* remove the file */
scan_file_remove(scan, file);
/* insert the file in the delayed block allocation */
scan_file_delayed_insert(scan, file);
/* reinsert the copy in the delayed list */
scan_file_insert(scan, copy);
}
}
/**
* Removes the specified file from the data set.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
/* remove the file from the file containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
}
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* deallocate */
file_free(file);
}
/**
* Inserts the specified file in the data set.
*/
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t block_max;
block_off_t block_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
block_pos = disk->first_free_block;
block_max = tommy_arrayblk_size(&disk->blockarr);
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = &file->blockvec[i];
/* increment the position until the first really free block */
while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
++block_pos;
/* if not found, allocate a new one */
if (block_pos == block_max) {
++block_max;
tommy_arrayblk_grow(&disk->blockarr, block_max);
}
/* set the position */
block->parity_pos = block_pos;
/* get block specific info */
info = info_get(&state->infoarr, block_pos);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(&file->blockvec[i], BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
struct snapraid_block* over_block;
unsigned block_state;
/* convert to a CHG block */
block_state_set(block, BLOCK_STATE_CHG);
/* block to overwrite */
over_block = tommy_arrayblk_get(&disk->blockarr, block_pos);
/* state of the block we are going to overwrite */
block_state = block_state_get(over_block);
/* if the block is an empty one */
if (block_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the past hash */
if (!state->clear_past_hash) {
/* in this case we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
tommy_arrayblk_set(&disk->blockarr, block_pos, block);
/* set the new free position */
disk->first_free_block = block_pos + 1;
}
/* note that the file is already added in the file hashtables */
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
/**
* Processes a file.
*/
@ -869,7 +882,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
/* if found, and it's a fully hashed file */
if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
/* assume that the file is a copy, and reuse the hash */
file_copy(other_file, file);
@ -909,13 +922,8 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
}
}
/* insert the file in the file hashtables, to allow to find duplicate hardlinks */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* insert the file in the delayed block allocation */
scan_file_delayed_insert(scan, file);
/* insert the file in the delayed list */
scan_file_insert(scan, file);
}
/**
@ -1544,8 +1552,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
/* next node */
node = node->next;
/* insert it */
scan_file_insert(scan, file);
/* insert in the parity */
scan_file_allocate(scan, file);
}
/* mark the disk without reliable physical offset if it has duplicates */

2
cmdline/scrub.c
View file

@ -200,7 +200,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
}
/* if the block is not used */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);

22
cmdline/snapraid.c
View file

@ -1001,9 +1001,29 @@ int main(int argc, char* argv[])
/* in the next state read ensures to clear all the past hashes in case */
/* we are reading from an incomplete sync */
/* The undeterminated hash are only for CHG/DELETED blocks for which we don't */
/* know if the previous interrupted sync was able to update or not the parity */
/* know if the previous interrupted sync was able to update or not the parity. */
/* The sync process instead needs to trust this information because it's used */
/* to avoid to recompute the parity if all the input are equals as before. */
/* In these cases we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the data that may be now */
/* deleted. Then we reset the hash to a bogus value. */
/* An example for CHG blocks is: */
/* - One file is added creating a CHG block with ZERO state */
/* - Sync aborted after updating the parity to the new state, */
/* but without saving the content file representing this new BLK state. */
/* - File is now deleted after the aborted sync */
/* - Sync again, deleting the blocks overt the CHG ones */
/* with the hash of CHG blocks not represeting the real parity state */
/* An example for DELETED blocks is: */
/* - One file is deleted creating DELETED blocks */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new EMPTY state. */
/* - Another file is added again over the DELETE ones */
/* with the hash of DELETED blocks not represeting the real parity state */
state.clear_past_hash = 1;
state_read(&state);

44
cmdline/state.c
View file

@ -1389,7 +1389,7 @@ static int position_is_required(struct snapraid_state* state, block_off_t pos)
/* check for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
/* if we have at least one file, the position is needed */
if (block_has_file(block))
@ -1418,7 +1418,7 @@ static int info_is_required(struct snapraid_state* state, block_off_t pos)
/* check for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
/* if we have at least one synced file, the info is required */
if (block_state_get(block) == BLOCK_STATE_BLK)
@ -1435,12 +1435,12 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos
/* check for each disk if block is really used */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
/* if the block is deleted */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* set it to empty */
tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY);
fs_par2block_clear(disk, pos);
}
}
}
@ -1450,7 +1450,7 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos
*/
static int is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
{
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
return block_state_get(block) == BLOCK_STATE_DELETED;
}
@ -1616,7 +1616,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
@ -1624,8 +1624,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* insert the block in the block array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + 1);
tommy_arrayblk_set(&disk->blockarr, v_pos, block);
fs_par2block_set(disk, v_pos, block);
/* check for termination of the block list */
++blockidx;
@ -1765,7 +1764,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
@ -1776,8 +1775,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* insert the block in the block array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + 1);
tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block);
fs_par2block_set(disk, v_pos, &deleted->block);
} else if (strcmp(tag, "file") == 0) {
/* file */
char sub[PATH_MAX];
@ -2756,7 +2754,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
unsigned block_state = block_state_get(block);
/* consistency check */
if (block != disk_block_get(disk, block->parity_pos)) {
if (block != fs_par2block_get(disk, block->parity_pos)) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub);
exit(EXIT_FAILURE);
@ -2858,7 +2856,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
}
for (b = 0; b < blockmax; ++b) {
if (is_block_deleted(disk, b)) {
struct snapraid_block* block = disk_block_get(disk, b);
struct snapraid_block* block = fs_par2block_get(disk, b);
/* consistency check */
if (block->parity_pos != b) {
@ -3190,9 +3188,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* LCOV_EXCL_START */
}
/* grow the array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count);
/* fill the blocks in the run */
while (v_count) {
struct snapraid_block* block = &file->blockvec[v_idx];
@ -3277,7 +3272,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Internal inconsistency in block existence!\n");
@ -3286,7 +3281,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* insert the block in the block array */
tommy_arrayblk_set(&disk->blockarr, v_pos, block);
fs_par2block_set(disk, v_pos, block);
/* go to the next block */
++v_idx;
@ -3437,9 +3432,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
switch (c) {
case 'o' :
/* grow the array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count);
/* if it's a run of deleted blocks */
while (v_count) {
struct snapraid_deleted* deleted;
@ -3466,7 +3458,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos);
if (state->opt.skip_content_check) {
@ -3482,7 +3474,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* insert the block in the block array */
tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block);
fs_par2block_set(disk, v_pos, &deleted->block);
}
/* go to next block */
@ -4176,7 +4168,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
struct snapraid_block* block = blockvec + b;
/* consistency check */
if (block != disk_block_get(disk, block->parity_pos)) {
if (block != fs_par2block_get(disk, block->parity_pos)) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub);
exit(EXIT_FAILURE);
@ -4275,7 +4267,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
/* write all the hash */
while (begin < end) {
struct snapraid_block* block = disk_block_get(disk, begin);
struct snapraid_block* block = fs_par2block_get(disk, begin);
/* consistency check */
if (block->parity_pos != begin) {
@ -4677,7 +4669,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom
if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
|| filter_path(filterlist_file, 0, disk->name, file->sub) != 0
|| filter_existence(filter_missing, disk->dir, file->sub) != 0
|| filter_correctness(filter_error, &state->infoarr, file) != 0
|| filter_correctness(filter_error, &state->infoarr, disk, file) != 0
) {
file_flag_set(file, FILE_IS_EXCLUDED);
}

12
cmdline/status.c
View file

@ -136,12 +136,13 @@ int state_status(struct snapraid_state* state)
/* check fragmentation */
if (file->blockmax != 0) {
block_off_t prev_pos;
block_off_t last_pos;
int fragmented;
fragmented = 0;
prev_pos = file->blockvec[0].parity_pos;
prev_pos = fs_file2par_get(disk, file, 0);
for (j = 1; j < file->blockmax; ++j) {
block_off_t parity_pos = file->blockvec[j].parity_pos;
block_off_t parity_pos = fs_file2par_get(disk, file, j);
if (prev_pos + 1 != parity_pos) {
fragmented = 1;
++extra_fragment;
@ -151,8 +152,9 @@ int state_status(struct snapraid_state* state)
}
/* keep track of latest block used */
if (file->blockvec[file->blockmax - 1].parity_pos > disk_block_latest_used) {
disk_block_latest_used = file->blockvec[file->blockmax - 1].parity_pos;
last_pos = fs_file2par_get(disk, file, file->blockmax - 1);
if (last_pos > disk_block_latest_used) {
disk_block_latest_used = last_pos;
}
if (fragmented) {
@ -279,7 +281,7 @@ int state_status(struct snapraid_state* state)
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
struct snapraid_block* block = fs_par2block_get(disk, i);
if (block_has_file(block))
one_valid = 1;

12
cmdline/sync.c
View file

@ -65,7 +65,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
struct snapraid_block* block;
unsigned block_state;
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -101,7 +101,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
unsigned char hash[HASH_SIZE];
unsigned block_state;
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -424,7 +424,7 @@ static int block_is_enabled(block_off_t i, struct snapraid_handle* handle, unsig
if (!handle[j].disk)
continue;
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
if (block_has_file(block))
one_valid = 1;
@ -584,7 +584,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
/* get the block */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -1029,7 +1029,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
for (j = 0; j < diskmax; ++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
if (block == BLOCK_EMPTY) {
/* nothing to do */
@ -1039,7 +1039,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* if it's a deleted block */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* the parity is now updated without this block, so it's now empty */
tommy_arrayblk_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
fs_par2block_clear(handle[j].disk, i);
continue;
}