rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Keeps track of the hash of deleted blocks.

Browse source 
This will allow an easier recovering providing manually deleted files.
This commit is contained in:
Andrea Mazzoleni 2013-02-24 19:02:53 +01:00
parent 027b6e3fcd
commit 5c4f6dc5c4
10 changed files with 357 additions and 159 deletions
Download patch file Download diff file Expand all files Collapse all files

70
Makefile.am
View file

@ -209,7 +209,7 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Abort sync with additions and then delete the new additions
echo --- Abort sync with additions and then delete the new additions and sync again
cp -pR test/disk1/_a test/disk1/_c
cp -pR test/disk2/_a test/disk2/_c
cp -pR test/disk3/_a test/disk3/_c
@ -219,53 +219,38 @@ endif
rm -r test/disk3/_c
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Abort sync late with more additions than deletion and then delete after
cp -pR test/disk1/_a test/disk1/_c
cp -pR test/disk1/_b test/disk1/_d
mv test/disk1/_a test/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
echo --- Abort sync late with additions and then delete some unchanged stuff and fix with RAID5
cp -pR test/disk1/_b test/disk1/_c
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf --test-kill-after-sync sync
rm -r test/disk2
mkdir test/disk2
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log
rm -r test/disk1/_c
rm -r test/disk1/_d
mv test/_a test/disk1/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Abort sync late with more deletions than additions and then delete after
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Abort sync late with more deletions than additions and then delete some unchanged stuff and fix with RAID5
cp -pR test/disk1/_a test/disk1/_c
mv test/disk1/_a test/_a
mv test/disk1/_b test/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
-rm -r test/disk1/_b
rm -r test/disk1/_c
mv test/_a test/disk1/_a
mv test/_b test/disk1/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Abort sync early with more additions than deletions and then delete after
cp -pR test/disk1/_a test/disk1/_c
cp -pR test/disk1/_b test/disk1/_d
mv test/disk1/_a test/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
rm -r test/disk1/_c
rm -r test/disk1/_d
mv test/_a test/disk1/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Abort sync early with more deletions than additions and then delete after
cp -pR test/disk1/_a test/disk1/_c
mv test/disk1/_a test/_a
mv test/disk1/_b test/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
rm -r test/disk2
mkdir test/disk2
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log
-rm -r test/disk1/_a
-rm -r test/disk1/_b
rm -r test/disk1/_c
mv test/_a test/disk1/_a
mv test/_b test/disk1/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Abort sync early with more deletions than additions and then delete some unchanged stuff and fix with RAID6
cp -pR test/disk1/_a test/disk1/_c
mv test/disk1/_a test/_a
mv test/disk1/_b test/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync
rm -r test/disk2
mkdir test/disk2
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-unrecoverable -c $(srcdir)/test-raid5.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
-rm -r test/disk1/_b
@ -273,6 +258,7 @@ endif
mv test/_a test/disk1/_a
mv test/_b test/disk1/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete some files, fix with -m and check with raid5
rm test/disk1/8*
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log

76
check.c
View file

@ -57,6 +57,7 @@ struct failed_struct {
/**
* Checks if a block hash matches the specified buffer.
* Return ==0 if equal
*/
static int blockcmp(struct snapraid_state* state, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero)
{
@ -106,8 +107,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
/* if there isn't, we have to sacrifice a parity block to check that the result is correct */
has_hash = 0;
for(j=0;j<failed_count;++j) {
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block)) /* if the block can be checked */
if (block_has_hash(failed[failed_map[j]].block)) /* if the block can be checked */
has_hash = 1;
}
@ -123,9 +123,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
/* check if the recovered blocks are OK */
for(j=0;j<1;++j) {
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block) /* if the block can be checked */
) {
if (block_has_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
@ -155,9 +153,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
/* check if the recovered blocks are OK */
for(j=0;j<1;++j) {
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block) /* if the block can be checked */
) {
if (block_has_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
@ -207,9 +203,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
/* check if the recovered blocks are OK */
for(j=0;j<2;++j) {
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block) /* if the block can be checked */
) {
if (block_has_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
@ -260,7 +254,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
has_bad = 1;
/* we never set a DELETED block as bad */
assert(failed[j].block != BLOCK_DELETED);
assert(block_state_get(failed[j].block) != BLOCK_STATE_DELETED);
failed_map[n] = j;
++n;
@ -286,9 +280,9 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
unsigned block_state = block_state_get(failed[j].block);
if (block_state == BLOCK_STATE_NEW) {
/* if the block is not filled with 0, we are sure to have restored it */
/* to the state after the 'sync' */
/* if the block is filled with 0, it could be either that the */
/* if the block is not filled with 0, we are sure to have */
/* restored it to the state after the 'sync' */
/* instead, if the block is filled with 0, it could be either that the */
/* block after the sync is really filled by 0, or that */
/* we restored the block before the 'sync'. */
if (memcmp(buffer[failed[j].index], buffer_zero, state->block_size) == 0) {
@ -298,10 +292,22 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
}
if (block_state == BLOCK_STATE_CHG) {
/* we cannot know if we recoverd the state before or after */
/* the partial 'sync' process. */
/* just to be safe, we assume it is garbage */
failed[j].is_outofdate = 1;
/* if the hash is a bogus value we cannot check the result */
/* this could happen if we have lost this information */
/* after an aborted sync */
if (memcmp(failed[j].block->hash, buffer_zero, HASH_SIZE) == 0) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
} else
/* if the hash is different the previous one, we are sure to have */
/* restored it to the state after the 'sync' */
/* instead, if the hash matches, it could be either that the */
/* block after the sync has this hash, or that */
/* we restored the block before the 'sync'. */
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
}
}
}
}
@ -322,10 +328,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
if (failed[j].is_bad)
has_bad = 1;
if (failed[j].block == BLOCK_DELETED)
block_state = BLOCK_STATE_DELETED;
else
block_state = block_state_get(failed[j].block);
block_state = block_state_get(failed[j].block);
if (failed[j].is_bad /* if the block is bad we don't know its content */
|| block_state == BLOCK_STATE_DELETED /* we don't know the original content of deleted blocks */
@ -435,8 +438,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block_is_valid(block)
&& block_has_hash(block) /* only if the block is hashed */
if (block_has_hash(block) /* only if the block is hashed */
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
@ -466,8 +468,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block_is_valid(block)
&& block_has_hash(block) /* only if the block is hashed */
if (block_has_hash(block) /* only if the block is hashed */
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
@ -509,11 +510,14 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
continue;
}
/* get the state of the block */
block_state = block_state_get(block);
/* if the block is deleted */
if (block == BLOCK_DELETED) {
if (block_state == BLOCK_STATE_DELETED) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
/* mark the parity as invalid, and don't try to check/fix it */
/* because it will be recomputed at the next sync */
/* note that for sure there is also another block invalidating */
@ -525,7 +529,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
failed[failed_count].is_bad = 0;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = BLOCK_DELETED;
failed[failed_count].block = block;
failed[failed_count].handle = 0;
++failed_count;
continue;
@ -540,9 +544,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
continue;
}
/* get the state of the block */
block_state = block_state_get(block);
/* if the parity is not valid */
if (block_state != BLOCK_STATE_BLK) {
/* mark the parity as invalid, and don't try to check/fix it */
@ -868,8 +869,9 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (!block_is_valid(block)) {
/* if no block, no file and nothing to do */
if (!block_has_file(block)) {
/* if no file, nothing to do */
continue;
}
@ -1094,12 +1096,12 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
fprintf(stderr, "Error stating hardlink-to '%s'. %s.\n", pathto, strerror(errno));
fprintf(stderr, "hardlinkerror:%s:%s:%s: Hardlink to stat error\n", disk->name, link->sub, link->linkto);
++error;
} else if (!S_ISREG(st.st_mode)) {
} else if (!S_ISREG(stto.st_mode)) {
failed = 1;
fprintf(stderr, "hardlinkerror:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, link->sub, link->linkto);
++error;
} else if (st.st_ino != stto.st_ino) {
} else if (!failed && st.st_ino != stto.st_ino) {
failed = 1;
fprintf(stderr, "Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto);

5
dry.c
View file

@ -57,8 +57,11 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (!block_is_valid(block))
if (!block_has_file(block)) {
/* if no file, nothing to do */
continue;
}
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {

32
elem.c
View file

@ -295,6 +295,36 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
return block_size;
}
struct snapraid_deleted* deleted_alloc(void)
{
struct snapraid_deleted* deleted;
deleted = malloc_nofail(sizeof(struct snapraid_deleted));
/* set the state as deleted */
deleted->block.file_mixed = BLOCK_STATE_DELETED;
return deleted;
}
struct snapraid_deleted* deleted_dup(struct snapraid_block* block)
{
struct snapraid_deleted* deleted;
deleted = deleted_alloc();
/* copy data from deleted block */
deleted->block.parity_pos = block->parity_pos;
memcpy(deleted->block.hash, block->hash, HASH_SIZE);
return deleted;
}
void deleted_free(struct snapraid_deleted* deleted)
{
free(deleted);
}
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode)
{
struct snapraid_file* file;
@ -415,6 +445,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
disk->device = dev;
disk->first_free_block = 0;
tommy_list_init(&disk->filelist);
tommy_list_init(&disk->deletedlist);
tommy_hashdyn_init(&disk->inodeset);
tommy_hashdyn_init(&disk->pathset);
tommy_list_init(&disk->linklist);
@ -429,6 +460,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
void disk_free(struct snapraid_disk* disk)
{
tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free);
tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)deleted_free);
tommy_hashdyn_done(&disk->inodeset);
tommy_hashdyn_done(&disk->pathset);
tommy_list_foreach(&disk->linklist, (tommy_foreach_func*)link_free);

159
elem.h
View file

@ -72,45 +72,63 @@ struct snapraid_file;
* This is the normal state of a saved block.
* Note that if exists at least one BLOCK_STATE_BLK in a disk, all the other
* blocks at the same address in other disks can be only BLOCK_EMPTY or BLOCK_STATE_BLK.
*
* The block hash field IS set.
* The parity for this disk is updated.
*/
#define BLOCK_STATE_BLK 0
#define BLOCK_STATE_BLK 1
/**
* The block is new and not yet hashed, and it's using a space previously empty.
*
* The block hash field IS NOT set.
* The parity for this disk is not updated, but it contains 0.
*/
#define BLOCK_STATE_NEW 1
#define BLOCK_STATE_NEW 2
/**
* The block has the hash computed, but the parity is invalid.
* This happens when the parity is invalidated by a change in another block.
* This happens when the parity is invalidated by a change in another disk.
*
* The block hash field IS set.
* The parity for this disk is updated, but there is at least another disk not updated parity.
*/
#define BLOCK_STATE_INV 2
#define BLOCK_STATE_INV 3
/**
* The block is new and not yet hashed, and it's using the space of a block not existing anymore.
* This happens when a new block overwrite a just removed block.
*
* The block hash field IS set, and it represents the hash of the previous data.
* The parity for this disk is not updated, but it contains the old data referenced by the hash.
*
* If the hash is completely filled with 0, it means that its lost.
*/
#define BLOCK_STATE_CHG 3
#define BLOCK_STATE_CHG 4
/**
* This block is a deleted one.
* This happens when a file is deleted.
*
* The block hash field IS set, and it represents the hash of the previous data.
* The parity for this disk is not updated, but it contains the old data referenced by the hash.
*
* If the hash is completely filled with 0, it means that its lost.
*/
#define BLOCK_STATE_DELETED 5
/**
* Mask used to store the previous states of the block.
* Used to mix them inside the file pointer.
*/
#define BLOCK_STATE_MASK 3
/**
* This block is a deleted one.
* Note that this state cannot be stored inside the block,
* but it's usually represented by the block ::BLOCK_DELETED.
*/
#define BLOCK_STATE_DELETED 4
#define BLOCK_STATE_MASK 7
/**
* This block is an empty one.
* Note that this state cannot be stored inside the block,
* but it's usually represented by the block ::BLOCK_EMPTY.
* and it's represented by the block ::BLOCK_EMPTY.
*/
#define BLOCK_STATE_EMPTY 5
#define BLOCK_STATE_EMPTY 8
/**
* Block of a file.
@ -126,11 +144,6 @@ struct snapraid_block {
*/
#define BLOCK_EMPTY 0
/**
* Block pointer used to mark deleted blocks.
*/
#define BLOCK_DELETED ((struct snapraid_block*)-1)
#define FILE_IS_PRESENT 1 /**< If it's seen as present. */
#define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */
#define FILE_IS_LARGER 4 /**< If a larger file was already detected. Just to avoid to report it more times. */
@ -188,6 +201,21 @@ struct snapraid_dir {
tommy_hashdyn_node nodeset;
};
/**
* Deleted entry.
*/
struct snapraid_deleted {
/**
* Deleted block.
* This block is always in state BLOCK_STATE_DELETED,
* and it's used to keep the old hash during a sync.
*/
struct snapraid_block block;
/* nodes for data structures */
tommy_node node;
};
/**
* Disk.
*/
@ -196,14 +224,33 @@ struct snapraid_disk {
char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
uint64_t device; /**< Device identifier. */
block_off_t first_free_block; /**< First free searching block. */
tommy_list filelist; /**< List of all the files. */
/**<
* Block array of the disk.
*
* Each element points to a snapraid_block structure, or it's BLOCK_EMPTY.
*/
tommy_array blockarr;
/**
* List of all the snapraid_file for the disk.
*/
tommy_list filelist;
/**
* List of all the snapraid_deleted blocks for the disk.
*
* These files are kept allocated, because the blocks are still referenced in
* the ::blockarr.
*/
tommy_list deletedlist;
tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
tommy_hashdyn pathset; /**< Hashtable by path of all the files. */
tommy_list linklist; /**< List of all the links. */
tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
tommy_list dirlist; /**< List of all the dirs. */
tommy_hashdyn dirset; /**< Hashtable by name of all the dirs. */
tommy_array blockarr; /**< Block array of the disk. */
/* nodes for data structures */
tommy_node node;
@ -297,17 +344,39 @@ static inline struct snapraid_file* block_file_get(struct snapraid_block* block)
*/
static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file)
{
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | (uintptr_t)file;
uintptr_t ptr = (uintptr_t)file;
/* ensure that the pointer doesn't use the flag space */
if ((ptr & (uintptr_t)BLOCK_STATE_MASK) != 0) {
fprintf(stderr, "Internal error for pointer not aligned\n");
exit(EXIT_FAILURE);
}
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | ptr;
}
/**
* Get the state of the block.
*
* For this function, it's allowed to pass a NULL block
* pointer than results in the BLOCK_STATE_EMPTY state.
*/
static inline unsigned block_state_get(const struct snapraid_block* block)
{
if (block == BLOCK_EMPTY)
return BLOCK_STATE_EMPTY;
return block->file_mixed & BLOCK_STATE_MASK;
}
static inline void block_state_set(struct snapraid_block* block, unsigned state)
{
assert(state != BLOCK_STATE_EMPTY && state != BLOCK_STATE_DELETED);
/* ensure that the state can be stored inside the file pointer */
if ((state & BLOCK_STATE_MASK) != state) {
fprintf(stderr, "Internal error when setting the block state %u\n", state);
exit(EXIT_FAILURE);
}
block->file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK;
block->file_mixed |= state & BLOCK_STATE_MASK;
}
@ -315,17 +384,36 @@ static inline void block_state_set(struct snapraid_block* block, unsigned state)
static inline void block_clear_parity(struct snapraid_block* block)
{
unsigned state = block_state_get(block);
if (state == BLOCK_STATE_BLK) {
block_state_set(block, BLOCK_STATE_INV);
}
}
/**
* Check if the specified block has a valid and updated hash.
*
* Note that blocks with hash of old data, like CHG and DELETED ones, are not considered.
*/
static inline int block_has_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV;
}
/**
* Check if the specified block has a valid file.
*
* It includes BLK/INV/NEW/CHG and excludes EMPTY/DELETED.
*/
static inline int block_has_file(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG;
}
/**
* Gets the relative position of a block inside the file.
*/
@ -336,20 +424,27 @@ block_off_t block_file_pos(struct snapraid_block* block);
*/
int block_is_last(struct snapraid_block* block);
/**
* Checks if the block is a real allocated block.
*/
static inline int block_is_valid(struct snapraid_block* block)
{
return block != BLOCK_EMPTY && block != BLOCK_DELETED;
}
/**
* Gets the size in bytes of the block.
* If it's the last block of a file it could be less than block_size.
*/
unsigned block_file_size(struct snapraid_block* block, unsigned block_size);
/**
* Allocates a deleted block.
*/
struct snapraid_deleted* deleted_alloc(void);
/**
* Allocates a deleted block from a real one.
*/
struct snapraid_deleted* deleted_dup(struct snapraid_block* block);
/**
* Frees a deleted block.
*/
void deleted_free(struct snapraid_deleted* deleted);
static inline int file_flag_has(const struct snapraid_file* file, unsigned mask)
{
return (file->flag & mask) == mask;

2
maketest.sh
View file

@ -11,7 +11,7 @@ if ! make lcov_reset check lcov_capture lcov_html; then
exit 1
fi
rm -r cov_release
rm -rf cov_release
mv cov cov_release
if ! make distclean; then

4
parity.c
View file

@ -37,10 +37,10 @@ block_off_t parity_size(struct snapraid_state* state)
/* start from the declared size */
block_off_t block = tommy_array_size(&disk->blockarr);
/* decrease the block until an allocated one */
/* decrease the block until an used one */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_is_valid(tommy_array_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file(tommy_array_get(&disk->blockarr, block - 1)))
--block;
/* get the highest value */

68
scan.c
View file

@ -161,25 +161,57 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
/* free all the blocks of the file */
for(i=0;i<file->blockmax;++i) {
block_off_t block_pos = file->blockvec[i].parity_pos;
struct snapraid_block* block = &file->blockvec[i];
block_off_t block_pos = block->parity_pos;
unsigned block_state;
tommy_node* j;
struct snapraid_deleted* deleted;
/* adjust the first free position */
if (disk->first_free_block > block_pos)
disk->first_free_block = block_pos;
/* set the block as deleted */
tommy_array_set(&disk->blockarr, block_pos, BLOCK_DELETED);
/* in case we scan after an aborted sync, */
/* we could get also intermediate states like inv/chg/new */
block_state = block_state_get(block);
switch (block_state) {
case BLOCK_STATE_BLK :
case BLOCK_STATE_INV :
/* we keep the hash making it an "old" hash, because the parity is still containing data for it */
break;
case BLOCK_STATE_CHG :
case BLOCK_STATE_NEW :
/* in these cases we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Them we reset the hash to a bogus value */
/* Note that this condition is possible only if: */
/* - new files added/modified */
/* - aborted sync, without saving the content file */
/* - files deleted after the aborted sync */
memset(block->hash, 0, HASH_SIZE);
break;
default:
fprintf(stderr, "Internal state inconsistency in scanning for block %u state %u\n", block->parity_pos, block_state);
exit(EXIT_FAILURE);
}
/* allocated a new deleted block from the block we are going to delete */
deleted = deleted_dup(block);
/* insert it in the list of deleted blocks */
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* set the deleted block in the block array */
tommy_array_set(&disk->blockarr, block_pos, &deleted->block);
/* invalidate the block of all the other disks */
for(j=state->disklist;j!=0;j=j->next) {
struct snapraid_disk* oth_disk = j->data;
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
if (block_is_valid(oth_block)) {
/* remove the parity info for this block */
block_clear_parity(oth_block);
}
/* remove the parity info for this block */
block_clear_parity(oth_block);
}
}
@ -209,9 +241,10 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
block_max = tommy_array_size(&disk->blockarr);
for(i=0;i<file->blockmax;++i) {
tommy_node* j;
struct snapraid_block* block;
/* find a free block */
while (block_pos < block_max && block_is_valid(tommy_array_get(&disk->blockarr, block_pos)))
while (block_pos < block_max && block_has_file(tommy_array_get(&disk->blockarr, block_pos)))
++block_pos;
/* if not found, allocate a new one */
@ -223,21 +256,26 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
/* set the position */
file->blockvec[i].parity_pos = block_pos;
/* set the state depending if the used block was EMPTY or DELETED */
if (tommy_array_get(&disk->blockarr, block_pos) == BLOCK_EMPTY)
/* block to overwrite */
block = tommy_array_get(&disk->blockarr, block_pos);
/* if the block is an empty one */
if (block == BLOCK_EMPTY) {
/* we just overwrite it with a NEW one */
block_state_set(&file->blockvec[i], BLOCK_STATE_NEW);
else
} else {
/* otherwise it's a DELETED one, that we convert in CHG keeping the hash */
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
memcpy(file->blockvec[i].hash, block->hash, HASH_SIZE);
}
/* invalidate the block of all the other disks */
for(j=state->disklist;j!=0;j=j->next) {
struct snapraid_disk* oth_disk = j->data;
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
if (block_is_valid(oth_block)) {
/* remove the parity info for this block */
block_clear_parity(oth_block);
}
/* remove the parity info for this block */
block_clear_parity(oth_block);
}
/* store in the disk map, after invalidating all the other blocks */

68
state.c
View file

@ -717,11 +717,9 @@ static void state_content_check(struct snapraid_state* state, const char* path)
if (block == BLOCK_EMPTY) {
/* not relevant */
} else if (block == BLOCK_DELETED) {
parity_is_invalid = 1;
} else {
unsigned state = block_state_get(block);
if (state == BLOCK_STATE_BLK) {
unsigned block_state = block_state_get(block);
if (block_state == BLOCK_STATE_BLK) {
parity_is_valid = 1;
} else {
parity_is_invalid = 1;
@ -892,8 +890,8 @@ void state_read(struct snapraid_state* state)
block->parity_pos = v_pos;
hash = oathash32(hash, v_pos);
/* read the hash only for some state */
if (tag[0] == 'b' || tag[0] == 'i') {
/* read the hash only for 'blk/inv/chg', and not for 'new' */
if (tag[0] != 'n') {
c = sgeteol(f);
if (c != ' ') {
fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
@ -909,9 +907,15 @@ void state_read(struct snapraid_state* state)
hash = oathashm(hash, block->hash, HASH_SIZE);
}
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
fprintf(stderr, "Internal inconsistency for 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
/* insert the block in the block array */
tommy_array_grow(&disk->blockarr, block->parity_pos + 1);
tommy_array_set(&disk->blockarr, block->parity_pos, block);
tommy_array_grow(&disk->blockarr, v_pos + 1);
tommy_array_set(&disk->blockarr, v_pos, block);
/* check for termination of the block list */
++blockidx;
@ -925,6 +929,7 @@ void state_read(struct snapraid_state* state)
} else if (strcmp(tag, "off") == 0) {
/* "off" command */
block_off_t v_pos;
struct snapraid_deleted* deleted;
hash = oathash8(hash, 'o');
@ -933,24 +938,45 @@ void state_read(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
/* allocate a new deleted block */
deleted = deleted_alloc();
/* insert it in the list of deleted blocks */
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
ret = sgetu32(f, &v_pos);
if (ret < 0) {
fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
deleted->block.parity_pos = v_pos;
hash = oathash32(hash, v_pos);
/* insert the block in the block array */
tommy_array_grow(&disk->blockarr, v_pos + 1);
/* read the hash */
c = sgeteol(f);
if (c != ' ') {
fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
/* deleted block must be empty blocks */
if (tommy_array_get(&disk->blockarr, v_pos) != 0) {
/* set the hash only if present */
ret = sgethex(f, deleted->block.hash, HASH_SIZE);
if (ret < 0) {
fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashm(hash, deleted->block.hash, HASH_SIZE);
/* we must not overwrite existing blocks */
if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) {
fprintf(stderr, "Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
tommy_array_set(&disk->blockarr, v_pos, BLOCK_DELETED);
/* insert the block in the block array */
tommy_array_grow(&disk->blockarr, v_pos + 1);
tommy_array_set(&disk->blockarr, v_pos, &deleted->block);
} else if (strcmp(tag, "file") == 0) {
/* file */
char sub[PATH_MAX];
@ -1639,12 +1665,15 @@ void state_write(struct snapraid_state* state)
sputsl("chg ", f);
hash = oathash8(hash, 'g');
break;
default:
fprintf(stderr, "Internal state inconsistency in saving for block %u state %u\n", block->parity_pos, block_state);
exit(EXIT_FAILURE);
}
sputu32(block->parity_pos, f);
hash = oathash32(hash, block->parity_pos);
if (block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_INV) {
if (block_state != BLOCK_STATE_NEW) {
sputc(' ', f);
sputhex(block->hash, HASH_SIZE, f);
hash = oathashm(hash, block->hash, HASH_SIZE);
@ -1712,6 +1741,7 @@ void state_write(struct snapraid_state* state)
++count_dir;
}
/* save all the delete blocks */
{
block_off_t k;
block_off_t blockmax;
@ -1723,7 +1753,7 @@ void state_write(struct snapraid_state* state)
/* for each deleted block in the disk */
for(k=0;k<blockmax;++k) {
struct snapraid_block* block = tommy_array_get(&disk->blockarr, k);
if (block == BLOCK_DELETED) {
if (block_state_get(block) == BLOCK_STATE_DELETED) {
if (first_deleted) {
first_deleted = 0;
@ -1744,6 +1774,10 @@ void state_write(struct snapraid_state* state)
sputu32(k, f);
hash = oathash32(hash, k);
sputc(' ', f);
sputhex(block->hash, HASH_SIZE, f);
hash = oathashm(hash, block->hash, HASH_SIZE);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -1965,9 +1999,9 @@ int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off
if (delta > 5 && countpos > 0) {
unsigned m, h;
data_off_t todo = countmax - countpos;
data_off_t to_do = countmax - countpos;
m = todo * delta / (60 * countpos);
m = to_do * delta / (60 * countpos);
h = m / 60;
m = m % 60;

32
sync.c
View file

@ -71,7 +71,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block_is_valid(block)
if (block_has_file(block)
&& block_state_get(block) != BLOCK_STATE_BLK
) {
one_invalid = 1;
@ -107,7 +107,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block_is_valid(block)
/* if the block is used, and it needs some update */
if (block_has_file(block)
&& block_state_get(block) != BLOCK_STATE_BLK
) {
one_invalid = 1;
@ -117,12 +119,14 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* if no invalid block skip */
if (!one_invalid) {
/* cleanup all the deleted blocks */
/* cleanup all the deleted blocks at this position */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block == BLOCK_DELETED) {
/* if the block is deleted */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* set it to empty */
tommy_array_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
@ -155,9 +159,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
continue;
}
/* if the disk block is not used */
/* if the block is not used */
block = disk_block_get(handle[j].disk, i);
if (!block_is_valid(block)) {
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
@ -290,8 +294,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
continue;
}
if (block == BLOCK_DELETED) {
/* the parity is not updated without this block, so it's now empty */
/* if it's a deleted block */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* the parity is now updated without this block, so it's now empty */
tommy_array_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
continue;
}
@ -370,14 +375,17 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
loaded_size = state->loaded_blockmax * (data_off_t)state->block_size;
/* remove all the deleted blocks over the new parity size */
/* we know for sure that after the parity size, there is no used block */
/* and then it's safe to lose any deleted block info */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* disk = i->data;
block_off_t diskblockmax = tommy_array_size(&disk->blockarr);
block_off_t block;
block_off_t block_pos;
for(block=blockmax;block<diskblockmax;++block) {
if (tommy_array_get(&disk->blockarr, block) == BLOCK_DELETED) {
tommy_array_set(&disk->blockarr, block, BLOCK_EMPTY);
for(block_pos=blockmax;block_pos<diskblockmax;++block_pos) {
struct snapraid_block* block = tommy_array_get(&disk->blockarr, block_pos);
if (block_state_get(block) == BLOCK_STATE_DELETED) {
tommy_array_set(&disk->blockarr, block_pos, BLOCK_EMPTY);
/* mark the state as needing write */
state->need_write = 1;