Change the interface for parity allocation

This will allow a more compact representation
of the parity allocation
This commit is contained in:
Andrea Mazzoleni 2015-10-01 18:19:48 +02:00
parent 2801634aa4
commit 0407c945db
15 changed files with 449 additions and 286 deletions

View file

@ -619,7 +619,7 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
continue;
}
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* if no file, nothing to do */
continue;
@ -819,7 +819,7 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct
if (!handle[j].disk)
continue;
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
/* try to recover all files, even the ones without hash */
/* because in some cases we can recover also them */
@ -943,7 +943,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
}
/* if the disk block is not used */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (block == BLOCK_EMPTY) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);

View file

@ -67,7 +67,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
continue;
}
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* if no file, nothing to do */

View file

@ -50,9 +50,11 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE);
struct snapraid_block* block = fs_file2block_get(disk, file, i);
if (!block_has_updated_hash(&file->blockvec[i])) {
memcpy(buf + i * HASH_SIZE, block->hash, HASH_SIZE);
if (!block_has_updated_hash(block)) {
free(buf);
free(hash);
return 0;

View file

@ -309,7 +309,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub)
return 1;
}
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file)
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file)
{
unsigned i;
@ -318,12 +318,8 @@ int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapr
/* check each block of the file */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block;
snapraid_info info;
block = &file->blockvec[i];
info = info_get(infoarr, block->parity_pos);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
snapraid_info info = info_get(infoarr, parity_pos);
/* if the file has a bad block, don't exclude it */
if (info_get_bad(info))
@ -444,7 +440,6 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
file->flag = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].parity_pos = POS_INVALID;
file->blockvec[i].file_mixed = 0;
@ -460,10 +455,40 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
return file;
}
struct snapraid_file* file_dup(struct snapraid_file* copy)
{
struct snapraid_file* file;
block_off_t i;
file = malloc_nofail(sizeof(struct snapraid_file));
file->sub = strdup_nofail(copy->sub);
file->size = copy->size;
file->blockmax = copy->blockmax;
file->mtime_sec = copy->mtime_sec;
file->mtime_nsec = copy->mtime_nsec;
file->inode = copy->inode;
file->physical = copy->physical;
file->flag = copy->flag;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].file_mixed = copy->blockvec[i].file_mixed;
file->blockvec[i].parity_pos = copy->blockvec[i].parity_pos;
memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE);
/* set the file to overwrite the copied one */
block_file_set(&file->blockvec[i], file);
}
return file;
}
void file_free(struct snapraid_file* file)
{
free(file->sub);
file->sub = 0;
free(file->blockvec);
file->blockvec = 0;
free(file);
}
@ -739,6 +764,71 @@ void disk_free(struct snapraid_disk* disk)
free(disk);
}
block_off_t disk_block_size(struct snapraid_disk* disk)
{
return tommy_arrayblk_size(&disk->blockarr);
}
void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos)
{
(void)disk;
assert(file_pos < file->blockmax);
file->blockvec[file_pos].parity_pos = parity_pos;
}
struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
{
struct snapraid_block* block = fs_par2block_get(disk, parity_pos);
if (block == BLOCK_EMPTY)
return 0;
if (file_pos)
*file_pos = block_file_pos(block);
return block_file_get(block);
}
block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
(void)disk;
assert(file_pos < file->blockmax);
return file->blockvec[file_pos].parity_pos;
}
struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
(void)disk;
assert(file_pos < file->blockmax);
return &file->blockvec[file_pos];
}
void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos)
{
tommy_arrayblk_grow(&disk->blockarr, pos + 1);
tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY);
}
void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block)
{
tommy_arrayblk_grow(&disk->blockarr, pos + 1);
tommy_arrayblk_set(&disk->blockarr, pos, block);
}
struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
{
if (parity_pos < tommy_arrayblk_size(&disk->blockarr))
return tommy_arrayblk_get(&disk->blockarr, parity_pos);
else
return BLOCK_EMPTY;
}
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax)
{
block_off_t i;

View file

@ -465,7 +465,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub);
* Filter a file if bad.
* Return !=0 if the file is correct and it should be excluded.
*/
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file);
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
/**
* Filters a dir using a list of filters.
@ -503,7 +503,7 @@ static inline void block_file_set(struct snapraid_block* block, struct snapraid_
/* LCOV_EXCL_STOP */
}
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | ptr;
block->file_mixed = (block->file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr;
}
/**
@ -700,12 +700,17 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
}
/**
* Allocates a file.
* Allocate a file.
*/
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
/**
* Deallocates a file.
* Duplicate a file.
*/
struct snapraid_file* file_dup(struct snapraid_file* copy);
/**
* Deallocate a file.
*/
void file_free(struct snapraid_file* file);
@ -884,16 +889,47 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
void disk_free(struct snapraid_disk* disk);
/**
* Gets a specific block of a disk.
* Returns 0 if the block is over the end of the disk or not used.
* Get the size of the disk in blocks.
*/
static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, block_off_t pos)
{
if (pos < tommy_arrayblk_size(&disk->blockarr))
return tommy_arrayblk_get(&disk->blockarr, pos);
else
return BLOCK_EMPTY;
}
block_off_t disk_block_size(struct snapraid_disk* disk);
/**
* Set a filesystem mapping between file and parity positions.
*/
void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
/**
* Get the file position from the parity position.
* Return 0 if no file is using it.
*/
struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
/**
* Get the parity position from the file position.
*/
block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
/**
* Get the block from the file position.
*/
struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
/**
* Clear a specific block.
*/
void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos);
/**
* Set the block from parity position.
* In case of holes, everything is initialized to 0.
*/
void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block);
/**
* Get the block from the parity position.
* Returns BLOCK_EMPTY==0 if the block is over the end of the disk or not used.
*/
struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos);
/**
* Check if a disk is totally empty and can be discarded from the content file.

View file

@ -66,7 +66,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
file->path = strdup_nofail(path);
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
buffer = malloc_nofail(block_size);
@ -98,7 +98,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
offset = 0;
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_import_block* block = &file->blockvec[i];
struct snapraid_import_block* block = &file->blockimp[i];
unsigned read_size = block_size;
if (read_size > size)
read_size = size;
@ -145,7 +145,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
void import_file_free(struct snapraid_import_file* file)
{
free(file->path);
free(file->blockvec);
free(file->blockimp);
free(file);
}

View file

@ -46,7 +46,7 @@ struct snapraid_import_block {
*/
struct snapraid_import_file {
data_off_t size; /**< Size of the file. */
struct snapraid_import_block* blockvec; /**< All the blocks of the file. */
struct snapraid_import_block* blockimp; /**< All the blocks of the file. */
block_off_t blockmax; /**< Number of blocks. */
char* path; /**< Full path of the file. */

View file

@ -37,12 +37,12 @@ block_off_t parity_allocated_size(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
block_off_t block = disk_block_size(disk);
/* decrease the block until an allocated one, but part of a file */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file(fs_par2block_get(disk, block - 1)))
--block;
/* get the highest value */
@ -64,10 +64,10 @@ block_off_t parity_used_size(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
block_off_t block = disk_block_size(disk);
/* decrease the block until an used one */
while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file_and_valid_parity(fs_par2block_get(disk, block - 1)))
--block;
/* get the highest value */
@ -95,7 +95,7 @@ int parity_is_invalid(struct snapraid_state* state)
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
struct snapraid_block* block = fs_par2block_get(disk, i);
if (block_has_file(block))
one_valid = 1;
@ -134,7 +134,8 @@ void parity_overflow(struct snapraid_state* state, data_off_t size)
struct snapraid_file* file = j->data;
if (file->blockmax > 0) {
if (file->blockvec[file->blockmax - 1].parity_pos >= blockalloc) {
block_off_t parity_pos = fs_file2par_get(disk, file, file->blockmax - 1);
if (parity_pos >= blockalloc) {
log_tag("outofparity:%s:%s\n", disk->name, esc(file->sub));
if (first) {
first = 0;

View file

@ -155,22 +155,116 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
}
/**
* Removes the specified file from the parity.
* Insert the specified file in the parity.
*/
static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t parity_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
parity_pos = disk->first_free_block;
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
/* increment the position until the first really free block */
while (block_has_file(fs_par2block_get(disk, parity_pos)))
++parity_pos;
/* set the position */
block->parity_pos = parity_pos;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(block, BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
struct snapraid_block* over_block;
unsigned block_state;
/* convert to a CHG block */
block_state_set(block, BLOCK_STATE_CHG);
/* block to overwrite */
over_block = fs_par2block_get(disk, parity_pos);
/* state of the block we are going to overwrite */
block_state = block_state_get(over_block);
/* if the block is an empty one */
if (block_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the past hash */
if (!state->clear_past_hash) {
/* in this case we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
fs_par2block_set(disk, parity_pos, block);
/* set the new free position */
disk->first_free_block = parity_pos + 1;
}
/* insert in the list of contained files */
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
/**
* Remove the specified file from the parity.
* The file is then inserted in the deleted set, and it should no be deallocated.
*/
static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
/* remove from the list of contained files */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* state changed */
state->need_write = 1;
/* free all the blocks of the file */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
block_off_t block_pos = block->parity_pos;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
unsigned block_state;
struct snapraid_deleted* deleted;
@ -178,8 +272,8 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
/* note that doing all the deletions before alllocations, */
/* first_free_block is always 0 and the "if" is never triggered */
/* but we keep this code anyway for completeness. */
if (disk->first_free_block > block_pos)
disk->first_free_block = block_pos;
if (disk->first_free_block > parity_pos)
disk->first_free_block = parity_pos;
/* allocated a new deleted block from the block we are going to delete */
deleted = deleted_dup(block);
@ -223,75 +317,13 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* set the deleted block in the block array */
tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block);
}
}
/**
* Checks if a file is completely formed of blocks with invalid parity,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has an invalid parity */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* exclude blocks with parity */
if (!block_has_invalid_parity(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* if rehash fails */
if (info_get_rehash(info))
return 0;
fs_par2block_set(disk, parity_pos, &deleted->block);
}
return 1;
file_free(file);
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has a hash */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* exclude blocks without hash */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* exclude blocks needing a rehash */
if (info_get_rehash(info))
return 0;
}
return 1;
}
static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file)
static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
@ -312,9 +344,76 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid
}
}
/* insert in the delayed list */
tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
}
/**
* Checks if a file is completely formed of blocks with invalid parity,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has an invalid parity */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
/* exclude blocks with parity */
if (!block_has_invalid_parity(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* if rehash fails */
if (info_get_rehash(info))
return 0;
}
return 1;
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has a hash */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(disk, file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
/* exclude blocks without hash */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* exclude blocks needing a rehash */
if (info_get_rehash(info))
return 0;
}
return 1;
}
/**
* Refresh the file info.
*
@ -399,7 +498,43 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
}
/**
* Keeps the file as it's (or with only a name/inode modification).
* Insert the file in the data set.
*/
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* insert the file in the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* delayed allocationg of the parity */
scan_file_delayed_allocate(scan, file);
}
/**
* Remove the file from the data set.
*
* File is then deleted.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* remove the file from the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
}
/**
* Keep the file as it's (or with only a name/inode modification).
*
* If the file is kept, nothing has to be done.
*
@ -414,139 +549,17 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* fil
struct snapraid_disk* disk = scan->disk;
/* if the file is full invalid, schedule a reinsert at later stage */
if (file_is_full_invalid_parity_and_stable(scan->state, file)) {
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
struct snapraid_file* copy = file_dup(file);
/* remove the file from the list */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* remove the file */
scan_file_remove(scan, file);
/* insert the file in the delayed block allocation */
scan_file_delayed_insert(scan, file);
/* reinsert the copy in the delayed list */
scan_file_insert(scan, copy);
}
}
/**
* Removes the specified file from the data set.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
/* remove the file from the file containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
}
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* deallocate */
file_free(file);
}
/**
* Inserts the specified file in the data set.
*/
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t block_max;
block_off_t block_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
block_pos = disk->first_free_block;
block_max = tommy_arrayblk_size(&disk->blockarr);
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = &file->blockvec[i];
/* increment the position until the first really free block */
while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
++block_pos;
/* if not found, allocate a new one */
if (block_pos == block_max) {
++block_max;
tommy_arrayblk_grow(&disk->blockarr, block_max);
}
/* set the position */
block->parity_pos = block_pos;
/* get block specific info */
info = info_get(&state->infoarr, block_pos);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(&file->blockvec[i], BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
struct snapraid_block* over_block;
unsigned block_state;
/* convert to a CHG block */
block_state_set(block, BLOCK_STATE_CHG);
/* block to overwrite */
over_block = tommy_arrayblk_get(&disk->blockarr, block_pos);
/* state of the block we are going to overwrite */
block_state = block_state_get(over_block);
/* if the block is an empty one */
if (block_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the past hash */
if (!state->clear_past_hash) {
/* in this case we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
tommy_arrayblk_set(&disk->blockarr, block_pos, block);
/* set the new free position */
disk->first_free_block = block_pos + 1;
}
/* note that the file is already added in the file hashtables */
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
/**
* Processes a file.
*/
@ -869,7 +882,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
/* if found, and it's a fully hashed file */
if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
/* assume that the file is a copy, and reuse the hash */
file_copy(other_file, file);
@ -909,13 +922,8 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
}
}
/* insert the file in the file hashtables, to allow to find duplicate hardlinks */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* insert the file in the delayed block allocation */
scan_file_delayed_insert(scan, file);
/* insert the file in the delayed list */
scan_file_insert(scan, file);
}
/**
@ -1544,8 +1552,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
/* next node */
node = node->next;
/* insert it */
scan_file_insert(scan, file);
/* insert in the parity */
scan_file_allocate(scan, file);
}
/* mark the disk without reliable physical offset if it has duplicates */

View file

@ -200,7 +200,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
}
/* if the block is not used */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);

View file

@ -1001,9 +1001,29 @@ int main(int argc, char* argv[])
/* in the next state read ensures to clear all the past hashes in case */
/* we are reading from an incomplete sync */
/* The undeterminated hash are only for CHG/DELETED blocks for which we don't */
/* know if the previous interrupted sync was able to update or not the parity */
/* know if the previous interrupted sync was able to update or not the parity. */
/* The sync process instead needs to trust this information because it's used */
/* to avoid to recompute the parity if all the input are equals as before. */
/* In these cases we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the data that may be now */
/* deleted. Then we reset the hash to a bogus value. */
/* An example for CHG blocks is: */
/* - One file is added creating a CHG block with ZERO state */
/* - Sync aborted after updating the parity to the new state, */
/* but without saving the content file representing this new BLK state. */
/* - File is now deleted after the aborted sync */
/* - Sync again, deleting the blocks overt the CHG ones */
/* with the hash of CHG blocks not represeting the real parity state */
/* An example for DELETED blocks is: */
/* - One file is deleted creating DELETED blocks */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new EMPTY state. */
/* - Another file is added again over the DELETE ones */
/* with the hash of DELETED blocks not represeting the real parity state */
state.clear_past_hash = 1;
state_read(&state);

View file

@ -1389,7 +1389,7 @@ static int position_is_required(struct snapraid_state* state, block_off_t pos)
/* check for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
/* if we have at least one file, the position is needed */
if (block_has_file(block))
@ -1418,7 +1418,7 @@ static int info_is_required(struct snapraid_state* state, block_off_t pos)
/* check for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
/* if we have at least one synced file, the info is required */
if (block_state_get(block) == BLOCK_STATE_BLK)
@ -1435,12 +1435,12 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos
/* check for each disk if block is really used */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
/* if the block is deleted */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* set it to empty */
tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY);
fs_par2block_clear(disk, pos);
}
}
}
@ -1450,7 +1450,7 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos
*/
static int is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
{
struct snapraid_block* block = disk_block_get(disk, pos);
struct snapraid_block* block = fs_par2block_get(disk, pos);
return block_state_get(block) == BLOCK_STATE_DELETED;
}
@ -1616,7 +1616,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
@ -1624,8 +1624,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* insert the block in the block array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + 1);
tommy_arrayblk_set(&disk->blockarr, v_pos, block);
fs_par2block_set(disk, v_pos, block);
/* check for termination of the block list */
++blockidx;
@ -1765,7 +1764,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
@ -1776,8 +1775,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* insert the block in the block array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + 1);
tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block);
fs_par2block_set(disk, v_pos, &deleted->block);
} else if (strcmp(tag, "file") == 0) {
/* file */
char sub[PATH_MAX];
@ -2756,7 +2754,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
unsigned block_state = block_state_get(block);
/* consistency check */
if (block != disk_block_get(disk, block->parity_pos)) {
if (block != fs_par2block_get(disk, block->parity_pos)) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub);
exit(EXIT_FAILURE);
@ -2858,7 +2856,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
}
for (b = 0; b < blockmax; ++b) {
if (is_block_deleted(disk, b)) {
struct snapraid_block* block = disk_block_get(disk, b);
struct snapraid_block* block = fs_par2block_get(disk, b);
/* consistency check */
if (block->parity_pos != b) {
@ -3190,9 +3188,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* LCOV_EXCL_START */
}
/* grow the array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count);
/* fill the blocks in the run */
while (v_count) {
struct snapraid_block* block = &file->blockvec[v_idx];
@ -3277,7 +3272,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Internal inconsistency in block existence!\n");
@ -3286,7 +3281,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* insert the block in the block array */
tommy_arrayblk_set(&disk->blockarr, v_pos, block);
fs_par2block_set(disk, v_pos, block);
/* go to the next block */
++v_idx;
@ -3437,9 +3432,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
switch (c) {
case 'o' :
/* grow the array */
tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count);
/* if it's a run of deleted blocks */
while (v_count) {
struct snapraid_deleted* deleted;
@ -3466,7 +3458,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos);
if (state->opt.skip_content_check) {
@ -3482,7 +3474,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* insert the block in the block array */
tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block);
fs_par2block_set(disk, v_pos, &deleted->block);
}
/* go to next block */
@ -4176,7 +4168,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
struct snapraid_block* block = blockvec + b;
/* consistency check */
if (block != disk_block_get(disk, block->parity_pos)) {
if (block != fs_par2block_get(disk, block->parity_pos)) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub);
exit(EXIT_FAILURE);
@ -4275,7 +4267,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
/* write all the hash */
while (begin < end) {
struct snapraid_block* block = disk_block_get(disk, begin);
struct snapraid_block* block = fs_par2block_get(disk, begin);
/* consistency check */
if (block->parity_pos != begin) {
@ -4677,7 +4669,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom
if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
|| filter_path(filterlist_file, 0, disk->name, file->sub) != 0
|| filter_existence(filter_missing, disk->dir, file->sub) != 0
|| filter_correctness(filter_error, &state->infoarr, file) != 0
|| filter_correctness(filter_error, &state->infoarr, disk, file) != 0
) {
file_flag_set(file, FILE_IS_EXCLUDED);
}

View file

@ -136,12 +136,13 @@ int state_status(struct snapraid_state* state)
/* check fragmentation */
if (file->blockmax != 0) {
block_off_t prev_pos;
block_off_t last_pos;
int fragmented;
fragmented = 0;
prev_pos = file->blockvec[0].parity_pos;
prev_pos = fs_file2par_get(disk, file, 0);
for (j = 1; j < file->blockmax; ++j) {
block_off_t parity_pos = file->blockvec[j].parity_pos;
block_off_t parity_pos = fs_file2par_get(disk, file, j);
if (prev_pos + 1 != parity_pos) {
fragmented = 1;
++extra_fragment;
@ -151,8 +152,9 @@ int state_status(struct snapraid_state* state)
}
/* keep track of latest block used */
if (file->blockvec[file->blockmax - 1].parity_pos > disk_block_latest_used) {
disk_block_latest_used = file->blockvec[file->blockmax - 1].parity_pos;
last_pos = fs_file2par_get(disk, file, file->blockmax - 1);
if (last_pos > disk_block_latest_used) {
disk_block_latest_used = last_pos;
}
if (fragmented) {
@ -279,7 +281,7 @@ int state_status(struct snapraid_state* state)
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
struct snapraid_block* block = fs_par2block_get(disk, i);
if (block_has_file(block))
one_valid = 1;

View file

@ -65,7 +65,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
struct snapraid_block* block;
unsigned block_state;
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -101,7 +101,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
unsigned char hash[HASH_SIZE];
unsigned block_state;
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -424,7 +424,7 @@ static int block_is_enabled(block_off_t i, struct snapraid_handle* handle, unsig
if (!handle[j].disk)
continue;
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
if (block_has_file(block))
one_valid = 1;
@ -584,7 +584,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
/* get the block */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -1029,7 +1029,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
for (j = 0; j < diskmax; ++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
if (block == BLOCK_EMPTY) {
/* nothing to do */
@ -1039,7 +1039,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* if it's a deleted block */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* the parity is now updated without this block, so it's now empty */
tommy_arrayblk_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
fs_par2block_clear(handle[j].disk, i);
continue;
}