rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Detects copied files and reuse hash data.

Browse source
This commit is contained in:
Andrea Mazzoleni 2014-02-12 20:02:35 +01:00
parent 3dadfb99e9
commit c420391493
9 changed files with 272 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

11
HISTORY
View file

@ -10,11 +10,12 @@ SnapRAID HISTORY
But any new data added will be protected if you are using
enough parity to fix both the silent error and at least
another potential error.
* In "sync" run a preliminary hashing step for all the new files,
to ensure to detect silent errors caused by the heavy machine usage
of the parity computation.
You can disable this additional hashing with the new "rawsync"
configuration option.
* Detect copied files from one disk to another and reuse the already
computed hash information to validate them in "sync".
Files are assumed copied if they matches the name, size and timestamp.
* Added a new "prehash" configuration option to run a preliminary hashing
step for all the new files to ensure to detect silent errors caused
by the heavy machine usage of the parity computation.
* In "fix", if a previous attempt was made, resulting in a
.unrecoverable file, uses this file as starting point for the
new attempt.

14
TODO
View file

@ -15,20 +15,6 @@ can improve speed in sync. That should increase cache locality,
becase we read the data two times for hash and and parity,
and if we manage to keep it in the cache, we should save time.
* Detect files moved from one disk to another. We won't be able to reuse
parity, but we can reuse the hash to check that the file was moved
correctly.
We can assume a file moved to another disk if it has the same name, size,
and timestamp. Directory and inode can be different.
This can be implemented with the REP blocks.
In synch, we can do the hash check of such blocks before any other operation,
before invalidating the parity of the moved files now converted in DELETED
blocks.
If an error is found, or we stop the sync, or we can use the parity of
DELETED blocks to recover the missing ones.
Missing blocks can be find by hash, using all the DELETED blocks with hash
as search base.
* Avoid to do optimized move operation if more files with
same timestamp/size are present.

74
elem.c
View file

@ -394,7 +394,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t
/* set an invalid hash */
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
hash_zero_set(file->blockvec[i].hash);
hash_invalid_set(file->blockvec[i].hash);
}
return file;
@ -413,7 +413,27 @@ void file_rename(struct snapraid_file* file, const char* sub)
file->sub = strdup_nofail(sub);
}
const char* file_name(struct snapraid_file* file)
void file_copy(struct snapraid_file* src_file, struct snapraid_file* dst_file)
{
block_off_t i;
if (src_file->size != dst_file->size) {
/* LCOV_EXCL_START */
fprintf(stderr, "Internal inconsistency in copy file with different size\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
for(i=0;i<dst_file->blockmax;++i) {
/* set a block with hash computed but without parity */
block_state_set(&dst_file->blockvec[i], BLOCK_STATE_REP);
/* copy the hash */
memcpy(dst_file->blockvec[i].hash, src_file->blockvec[i].hash, HASH_SIZE);
}
}
const char* file_name(const struct snapraid_file* file)
{
const char* r = strrchr(file->sub, '/');
if (!r)
@ -445,7 +465,7 @@ int file_inode_compare(const void* void_a, const void* void_b)
return 0;
}
int file_alpha_compare(const void* void_a, const void* void_b)
int file_path_compare(const void* void_a, const void* void_b)
{
const struct snapraid_file* file_a = void_a;
const struct snapraid_file* file_b = void_b;
@ -463,13 +483,57 @@ int file_physical_compare(const void* void_a, const void* void_b)
return 0;
}
int file_path_compare(const void* void_arg, const void* void_data)
int file_path_compare_to_arg(const void* void_arg, const void* void_data)
{
const char* arg = void_arg;
const struct snapraid_file* file = void_data;
return strcmp(arg, file->sub);
}
int file_name_compare(const void* void_a, const void* void_b)
{
const struct snapraid_file* file_a = void_a;
const struct snapraid_file* file_b = void_b;
const char* name_a = file_name(file_a);
const char* name_b = file_name(file_b);
return strcmp(name_a, name_b);
}
int file_stamp_compare(const void* void_a, const void* void_b)
{
const struct snapraid_file* file_a = void_a;
const struct snapraid_file* file_b = void_b;
if (file_a->size < file_b->size)
return -1;
if (file_a->size > file_b->size)
return 1;
if (file_a->mtime_sec < file_b->mtime_sec)
return -1;
if (file_a->mtime_sec > file_b->mtime_sec)
return 1;
if (file_a->mtime_nsec < file_b->mtime_nsec)
return -1;
if (file_a->mtime_nsec > file_b->mtime_nsec)
return 1;
return 0;
}
int file_namestamp_compare(const void* void_a, const void* void_b)
{
int ret;
ret = file_name_compare(void_a, void_b);
if (ret != 0)
return ret;
return file_stamp_compare(void_a, void_b);
}
struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag)
{
struct snapraid_link* link;
@ -549,6 +613,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
tommy_list_init(&disk->deletedlist);
tommy_hashdyn_init(&disk->inodeset);
tommy_hashdyn_init(&disk->pathset);
tommy_hashdyn_init(&disk->stampset);
tommy_list_init(&disk->linklist);
tommy_hashdyn_init(&disk->linkset);
tommy_list_init(&disk->dirlist);
@ -564,6 +629,7 @@ void disk_free(struct snapraid_disk* disk)
tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)deleted_free);
tommy_hashdyn_done(&disk->inodeset);
tommy_hashdyn_done(&disk->pathset);
tommy_hashdyn_done(&disk->stampset);
tommy_list_foreach(&disk->linklist, (tommy_foreach_func*)link_free);
tommy_hashdyn_done(&disk->linkset);
tommy_list_foreach(&disk->dirlist, (tommy_foreach_func*)dir_free);

28
elem.h
View file

@ -236,6 +236,7 @@ struct snapraid_file {
tommy_node nodelist;
tommy_hashdyn_node nodeset;
tommy_hashdyn_node pathset;
tommy_hashdyn_node stampset;
};
/**
@ -321,6 +322,7 @@ struct snapraid_disk {
tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
tommy_hashdyn pathset; /**< Hashtable by path of all the files. */
tommy_hashdyn stampset; /**< Hashtable by stamp (size and time) of all the files. */
tommy_list linklist; /**< List of all the links. */
tommy_hashdyn linkset; /**< Hashtable by name of all the links. */
tommy_list dirlist; /**< List of all the dirs. */
@ -644,10 +646,15 @@ void file_free(struct snapraid_file* file);
*/
void file_rename(struct snapraid_file* file, const char* sub);
/**
* Copy a file.
*/
void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file);
/**
* Returns the name of the file, without the dir.
*/
const char* file_name(struct snapraid_file* file);
const char* file_name(const struct snapraid_file* file);
/**
* Compares a file with an inode.
@ -662,7 +669,7 @@ int file_inode_compare(const void* void_a, const void* void_b);
/**
* Compares files by path.
*/
int file_alpha_compare(const void* void_a, const void* void_b);
int file_path_compare(const void* void_a, const void* void_b);
/**
* Compares files by physical address.
@ -680,7 +687,14 @@ static inline tommy_uint32_t file_inode_hash(uint64_t inode)
/**
* Compares a file with a path.
*/
int file_path_compare(const void* void_arg, const void* void_data);
int file_path_compare_to_arg(const void* void_arg, const void* void_data);
/**
* Compare a file with another file for name, stamp, and both.
*/
int file_name_compare(const void* void_a, const void* void_b);
int file_stamp_compare(const void* void_a, const void* void_b);
int file_namestamp_compare(const void* void_a, const void* void_b);
/**
* Computes the hash of a file path.
@ -690,6 +704,14 @@ static inline tommy_uint32_t file_path_hash(const char* sub)
return tommy_hash_u32(0, sub, strlen(sub));
}
/**
* Computes the hash of a file stamp.
*/
static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec)
{
return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec)));
}
static inline int link_flag_has(const struct snapraid_link* link, unsigned mask)
{
return (link->flag & mask) == mask;

2
list.c
View file

@ -45,7 +45,7 @@ void state_list(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* sort by name */
tommy_list_sort(&disk->filelist, file_alpha_compare);
tommy_list_sort(&disk->filelist, file_path_compare);
/* for each file */
for(j=disk->filelist;j!=0;j=j->next) {

207
scan.c
View file

@ -22,6 +22,9 @@
#include "state.h"
struct snapraid_scan {
struct snapraid_state* state; /**< State used. */
struct snapraid_disk* disk; /**< Disk used. */
/**
* Counters of changes.
*/
@ -43,8 +46,11 @@ struct snapraid_scan {
/**
* Removes the specified link from the data set.
*/
static void scan_link_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_link* link)
static void scan_link_remove(struct snapraid_scan* scan, struct snapraid_link* link)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
@ -59,8 +65,11 @@ static void scan_link_remove(struct snapraid_state* state, struct snapraid_disk*
/**
* Inserts the specified link in the data set.
*/
static void scan_link_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_link* link)
static void scan_link_insert(struct snapraid_scan* scan, struct snapraid_link* link)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
@ -72,8 +81,10 @@ static void scan_link_insert(struct snapraid_state* state, struct snapraid_disk*
/**
* Processes a symbolic link.
*/
static void scan_link(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* sub, const char* linkto, unsigned link_flag)
static void scan_link(struct snapraid_scan* scan, int output, const char* sub, const char* linkto, unsigned link_flag)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
struct snapraid_link* link;
/* check if the link already exists */
@ -144,8 +155,11 @@ static void scan_link(struct snapraid_scan* scan, struct snapraid_state* state,
/**
* Removes the specified file from the parity.
*/
static void scan_file_deallocate(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
/* state changed */
@ -237,6 +251,38 @@ static int file_is_full_invalid(struct snapraid_file* file)
return 1;
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash or bad blocks is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_healthy(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* if no block, the file is valid */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for(i=0;i<file->blockmax;++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* if not hashed, fail */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* if rehash or bad, fail */
if (info_get_rehash(info) || info_get_bad(info))
return 0;
}
return 1;
}
/**
* Keeps the file as it's (or with only a name/inode modification).
*
@ -246,14 +292,16 @@ static int file_is_full_invalid(struct snapraid_file* file)
* This could happen after a failed sync, when some other files are deleted,
* and then new ones can be moved to fill the hole created.
*/
static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* if the file has some valid block, keep it where it is */
if (!file_is_full_invalid(file))
return;
/* deallocate the file from the parity */
scan_file_deallocate(state, disk, file);
scan_file_deallocate(scan, file);
/* remove the file from the list */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
@ -265,16 +313,19 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_state* st
/**
* Removes the specified file from the data set.
*/
static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* deallocate the file from the parity */
scan_file_deallocate(state, disk, file);
scan_file_deallocate(scan, file);
/* remove the file from the file containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
}
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* deallocate */
@ -284,8 +335,10 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
/**
* Inserts the specified file in the data set.
*/
static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t block_max;
block_off_t block_pos;
@ -297,6 +350,9 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
block_pos = disk->first_free_block;
block_max = tommy_arrayblk_size(&disk->blockarr);
for(i=0;i<file->blockmax;++i) {
snapraid_info info;
struct snapraid_block* block = &file->blockvec[i];
/* increment the position until the first really free block */
while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
++block_pos;
@ -308,44 +364,44 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
}
/* set the position */
file->blockvec[i].parity_pos = block_pos;
block->parity_pos = block_pos;
/* if the file block already has an updated hash */
if (block_has_updated_hash(&file->blockvec[i])) {
/* get block specific info */
info = info_get(&state->infoarr, block_pos);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(&file->blockvec[i]) == BLOCK_STATE_REP);
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(&file->blockvec[i], BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
struct snapraid_block* block;
struct snapraid_block* over_block;
unsigned block_state;
/* the only possible case is for CHG blocks */
assert(block_state_get(&file->blockvec[i]) == BLOCK_STATE_CHG);
/* convert to a CHG block */
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
block_state_set(block, BLOCK_STATE_CHG);
/* block to overwrite */
block = tommy_arrayblk_get(&disk->blockarr, block_pos);
over_block = tommy_arrayblk_get(&disk->blockarr, block_pos);
/* state of the block we are going to overwrite */
block_state = block_state_get(block);
block_state = block_state_get(over_block);
/* if the block is an empty one */
if (block_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(file->blockvec[i].hash);
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(file->blockvec[i].hash, block->hash, HASH_SIZE);
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the undeterminated hash */
if (!state->clear_undeterminate_hash) {
@ -359,13 +415,13 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(file->blockvec[i].hash);
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
tommy_arrayblk_set(&disk->blockarr, block_pos, &file->blockvec[i]);
tommy_arrayblk_set(&disk->blockarr, block_pos, block);
/* set the new free position */
disk->first_free_block = block_pos + 1;
@ -378,9 +434,12 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
/**
* Processes a file.
*/
static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* sub, const struct stat* st, uint64_t physical)
static void scan_file(struct snapraid_scan* scan, int output, const char* sub, const struct stat* st, uint64_t physical)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
struct snapraid_file* file;
tommy_node* i;
/*
* If the disk has persistent inodes and UUID, try a search on the past inodes,
@ -431,7 +490,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
if (file_flag_has(file, FILE_IS_PRESENT)) {
if (st->st_nlink > 1) {
/* it's a hardlink */
scan_link(scan, state, output, disk, sub, file->sub, FILE_IS_HARDLINK);
scan_link(scan, output, sub, file->sub, FILE_IS_HARDLINK);
return;
} else {
/* LCOV_EXCL_START */
@ -485,7 +544,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
}
/* mark the file as kept */
scan_file_keep(scan, state, disk, file);
scan_file_keep(scan, file);
/* nothing more to do */
return;
@ -525,7 +584,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
}
/* then try findind it by name */
file = tommy_hashdyn_search(&disk->pathset, file_path_compare, sub, file_path_hash(sub));
file = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub, file_path_hash(sub));
if (file) {
/* if the file is without an inode */
if (file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
@ -616,7 +675,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
}
/* mark the file as kept */
scan_file_keep(scan, state, disk, file);
scan_file_keep(scan, file);
/* nothing more to do */
return;
@ -650,7 +709,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
}
/* remove it */
scan_file_remove(state, disk, file);
scan_file_remove(scan, file);
/* and continue to reinsert it */
} else {
@ -671,9 +730,34 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
/* mark it as present */
file_flag_set(file, FILE_IS_PRESENT);
/* now search for a file with the same name and timestamp in all the disks */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* other_disk = i->data;
struct snapraid_file* other_file;
tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
/* search for a file with the same name and stamp */
other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
/* if found, and it's a fully hashed file */
if (other_file && file_is_full_hashed_and_healthy(scan->state, other_file)) {
/* assume that the file is a copy, and reuse the hash */
file_copy(other_file, file);
fprintf(stdlog, "scan:copy:%s:%s:%s:%s\n", other_disk->name, other_file->sub, disk->name, file->sub);
if (output) {
printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub);
}
/* no need to continue the search */
break;
}
}
/* insert the file in the file hashtables, to allow to find duplicate hardlinks */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* insert the file in the delayed block allocation */
tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
@ -682,8 +766,11 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
/**
* Removes the specified dir from the data set.
*/
static void scan_emptydir_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_dir* dir)
static void scan_emptydir_remove(struct snapraid_scan* scan, struct snapraid_dir* dir)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
@ -698,8 +785,11 @@ static void scan_emptydir_remove(struct snapraid_state* state, struct snapraid_d
/**
* Inserts the specified dir in the data set.
*/
static void scan_emptydir_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_dir* dir)
static void scan_emptydir_insert(struct snapraid_scan* scan, struct snapraid_dir* dir)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
/* state changed */
state->need_write = 1;
@ -711,8 +801,9 @@ static void scan_emptydir_insert(struct snapraid_state* state, struct snapraid_d
/**
* Processes a dir.
*/
static void scan_emptydir(struct snapraid_scan* scan, struct snapraid_disk* disk, const char* sub)
static void scan_emptydir(struct snapraid_scan* scan, const char* sub)
{
struct snapraid_disk* disk = scan->disk;
struct snapraid_dir* dir;
/* check if the dir already exists */
@ -803,8 +894,10 @@ struct stat* dstat(const char* file, struct stat* st)
* Processes a directory.
* Return != 0 if at least one file or link is processed.
*/
static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* dir, const char* sub)
static int scan_dir(struct snapraid_scan* scan, int output, const char* dir, const char* sub)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
int processed = 0;
DIR* d;
tommy_list list;
@ -985,7 +1078,7 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
physical = 0;
}
scan_file(scan, state, output, disk, sub_next, st, physical);
scan_file(scan, output, sub_next, st, physical);
processed = 1;
} else {
if (state->opt.verbose) {
@ -1015,7 +1108,7 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
subnew[ret] = 0;
/* process as a symbolic link */
scan_link(scan, state, output, disk, sub_next, subnew, FILE_IS_SYMLINK);
scan_link(scan, output, sub_next, subnew, FILE_IS_SYMLINK);
processed = 1;
} else {
if (state->opt.verbose) {
@ -1041,9 +1134,9 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
pathslash(path_next, sizeof(path_next));
pathcpy(sub_dir, sizeof(sub_dir), sub_next);
pathslash(sub_dir, sizeof(sub_dir));
if (scan_dir(scan, state, output, disk, path_next, sub_dir) == 0) {
if (scan_dir(scan, output, path_next, sub_dir) == 0) {
/* scan the directory as empty dir */
scan_emptydir(scan, disk, sub_next);
scan_emptydir(scan, sub_next);
}
/* or we processed something internally, or we have added the empty dir */
processed = 1;
@ -1087,18 +1180,17 @@ void state_scan(struct snapraid_state* state, int output)
if (output)
printf("Comparing...\n");
/* first scan all the directory and find new and deleted files */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_scan* scan;
tommy_node* node;
int ret;
int has_persistent_inode;
unsigned phy_count;
unsigned phy_dup;
uint64_t phy_last;
struct snapraid_file* phy_file_last;
scan = malloc_nofail(sizeof(struct snapraid_scan));
scan->state = state;
scan->disk = disk;
scan->count_equal = 0;
scan->count_move = 0;
scan->count_restore = 0;
@ -1149,7 +1241,22 @@ void state_scan(struct snapraid_state* state, int output)
}
}
scan_dir(scan, state, output, disk, disk->dir, "");
scan_dir(scan, output, disk->dir, "");
}
/* we split the search in two phases because to detect files */
/* moved from one disk to another we have to start deletion */
/* only when all disks have all the new files found */
/* now process all the new and deleted files */
for(i=scanlist;i!=0;i=i->next) {
struct snapraid_scan* scan = i->data;
struct snapraid_disk* disk = scan->disk;
tommy_node* node;
unsigned phy_count;
unsigned phy_dup;
uint64_t phy_last;
struct snapraid_file* phy_file_last;
/* check for removed files */
node = disk->filelist;
@ -1168,7 +1275,7 @@ void state_scan(struct snapraid_state* state, int output)
printf("remove %s%s\n", disk->dir, file->sub);
}
scan_file_remove(state, disk, file);
scan_file_remove(scan, file);
}
}
@ -1189,7 +1296,7 @@ void state_scan(struct snapraid_state* state, int output)
printf("remove %s%s\n", disk->dir, link->sub);
}
scan_link_remove(state, disk, link);
scan_link_remove(scan, link);
}
}
@ -1203,7 +1310,7 @@ void state_scan(struct snapraid_state* state, int output)
/* remove if not present */
if (!dir_flag_has(dir, FILE_IS_PRESENT)) {
scan_emptydir_remove(state, disk, dir);
scan_emptydir_remove(scan, dir);
}
}
@ -1218,7 +1325,7 @@ void state_scan(struct snapraid_state* state, int output)
tommy_list_sort(&scan->file_insert_list, file_inode_compare);
break;
case SORT_ALPHA :
tommy_list_sort(&scan->file_insert_list, file_alpha_compare);
tommy_list_sort(&scan->file_insert_list, file_path_compare);
break;
case SORT_DIR :
/* already in order */
@ -1253,7 +1360,7 @@ void state_scan(struct snapraid_state* state, int output)
node = node->next;
/* insert it */
scan_file_insert(state, disk, file);
scan_file_insert(scan, file);
}
/* mark the disk without reliable physical offset if it has duplicates */
@ -1271,7 +1378,7 @@ void state_scan(struct snapraid_state* state, int output)
node = node->next;
/* insert it */
scan_link_insert(state, disk, link);
scan_link_insert(scan, link);
}
/* insert all the new dirs */
@ -1283,7 +1390,7 @@ void state_scan(struct snapraid_state* state, int output)
node = node->next;
/* insert it */
scan_emptydir_insert(state, disk, dir);
scan_emptydir_insert(scan, dir);
}
}

2
snapraid.c
View file

@ -422,7 +422,7 @@ int main(int argc, char* argv[])
break;
case OPT_TEST_SKIP_DEVICE :
opt.skip_device = 1;
period = 100; /* reduce period of the speed test */
period = 50; /* reduce period of the speed test */
break;
case OPT_TEST_FORCE_MURMUR3 :
opt.force_murmur3 = 1;

2
state.c
View file

@ -1594,6 +1594,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
/* insert the file in the file containers */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
/* start the block allocation of the file */
@ -2681,6 +2682,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* insert the file in the file containers */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
/* reads all the blocks */

19
sync.c
View file

@ -176,10 +176,10 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
|| handle[j].st.st_ino != block_file_get(block)->inode
) {
fprintf(stdlog, "error:%u:%s:%s: Unexpected change\n", i, handle[j].disk->name, handle[j].file->sub);
if (handle[j].st.st_size != block_file_get(block)->size)
fprintf(stdlog, "error:%u:%s:%s: Unexpected attribute change\n", i, handle[j].disk->name, handle[j].file->sub);
if (handle[j].st.st_size != block_file_get(block)->size) {
fprintf(stderr, "Unexpected size change at file '%s' from %"PRIu64" to %"PRIu64".\n", handle[j].path, block_file_get(block)->size, handle[j].st.st_size);
else if (handle[j].st.st_mtime != block_file_get(block)->mtime_sec
} else if (handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec) {
fprintf(stderr, "Unexpected time change at file '%s' from %"PRIu64".%d to %"PRIu64".%d.\n", handle[j].path, block_file_get(block)->mtime_sec, block_file_get(block)->mtime_nsec, (uint64_t)handle[j].st.st_mtime, (uint32_t)STAT_NSEC(&handle[j].st));
} else {
@ -589,10 +589,10 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
|| handle[j].st.st_ino != block_file_get(block)->inode
) {
fprintf(stdlog, "error:%u:%s:%s: Unexpected change\n", i, handle[j].disk->name, handle[j].file->sub);
if (handle[j].st.st_size != block_file_get(block)->size)
fprintf(stdlog, "error:%u:%s:%s: Unexpected attribute change\n", i, handle[j].disk->name, handle[j].file->sub);
if (handle[j].st.st_size != block_file_get(block)->size) {
fprintf(stderr, "Unexpected size change at file '%s' from %"PRIu64" to %"PRIu64".\n", handle[j].path, block_file_get(block)->size, handle[j].st.st_size);
else if (handle[j].st.st_mtime != block_file_get(block)->mtime_sec
} else if (handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec) {
fprintf(stderr, "Unexpected time change at file '%s' from %"PRIu64".%d to %"PRIu64".%d.\n", handle[j].path, block_file_get(block)->mtime_sec, block_file_get(block)->mtime_nsec, (uint64_t)handle[j].st.st_mtime, (uint32_t)STAT_NSEC(&handle[j].st));
} else {
@ -639,9 +639,10 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* if the file has invalid parity, it's a REP changed during the sync */
if (block_has_invalid_parity(block)) {
fprintf(stdlog, "error:%u:%s:%s: Unexpected change\n", i, handle[j].disk->name, handle[j].file->sub);
fprintf(stderr, "WARNING! You cannot modify files during a sync.\n");
fprintf(stderr, "Rerun the sync command when finished.\n");
fprintf(stdlog, "error:%u:%s:%s: Unexpected data change\n", i, handle[j].disk->name, handle[j].file->sub);
fprintf(stderr, "Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
fprintf(stderr, "WARNING! Unexpected data modification of a file without parity!\n");
fprintf(stderr, "Try removing the file from the array and rerun the 'sync' command!\n");
++error;