rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'maptree'

Browse source 
Conflicts:
	cmdline/elem.h
This commit is contained in:
Andrea Mazzoleni 2015-10-05 22:54:31 +02:00
commit e60fafa20e
31 changed files with 1876 additions and 2566 deletions
Download patch file Download diff file Expand all files Collapse all files

3
HISTORY
View file

@ -14,6 +14,9 @@ SnapRAID HISTORY
parity during sync is really correct.
The "status" command reports the amount of scrub necessary to
ensure to scrub all the synced blocks.
* Reduced the RAM memory usage, expecially with big files.
For each block, it now uses 17 bytes, instead of 28 bytes
(for 32 bit) and 36 bytes (for 64 bit).
* The -p, --percentage option supports the new the arguments:
"bad", "sync" and "full" to define the kind of scrub to do.
* Extends the -h, --pre-hash option to also verify moved and

41
Makefile.am
View file

@ -59,6 +59,7 @@ noinst_HEADERS = \
tommyds/tommyhash.h \
tommyds/tommyhashdyn.h \
tommyds/tommylist.h \
tommyds/tommytree.h \
tommyds/tommytypes.h \
tommyds/tommyarray.c \
tommyds/tommyarrayblk.c \
@ -66,6 +67,7 @@ noinst_HEADERS = \
tommyds/tommyhash.c \
tommyds/tommyhashdyn.c \
tommyds/tommylist.c \
tommyds/tommytree.c \
cmdline/portable.h \
cmdline/snapraid.h \
cmdline/util.h \
@ -225,6 +227,33 @@ if HAVE_SYMLINK
endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -a check
echo -- Filesystem allocation test
head -c 8192 /dev/zero > bench/disk1/TEST
head -c 8192 /dev/zero > bench/disk2/TEST
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync
# Now delete the file, and do a partial sync to force a deallocation in the middle of a file
rm bench/disk1/TEST
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E -S 1 -B 1 sync
# Now force a deallocation at the end
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E -S 7 -B 1 sync
rm bench/disk2/TEST
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E sync
echo -- Filesystem fragmentation test
head -c 8192 /dev/zero > bench/disk1/TEST1
head -c 8192 /dev/zero > bench/disk1/TEST2
head -c 8192 /dev/zero > bench/disk1/TEST3
head -c 8192 /dev/zero > bench/disk1/TEST4
head -c 8192 /dev/zero > bench/disk1/TEST5
head -c 8192 /dev/zero > bench/disk1/TEST6
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync
# Now delete some files, and create a bigger one to fill the holes
rm bench/disk1/TEST1
rm bench/disk1/TEST3
rm bench/disk1/TEST5
head -c 65536 /dev/zero > bench/disk1/TESTX
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync
rm -r bench/disk1/TEST*
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E sync
echo --- Create some files
$(TESTENV) ./mktest$(EXEEXT) generate 1 6 $(CHECKCOUNT) $(CHECKSIZE)
echo DUP > bench/disk1/DUP1
@ -532,23 +561,17 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR6) fix -l test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync
echo --- Delete some files and create some new, sync and check in multiple steps using text content
echo --- Delete some files and create some new, sync and check in multiple steps
rm bench/disk1/a/4*
rm bench/disk2/a/5*
rm bench/disk3/a/6*
rm bench/disk4/a/6*
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 10 sync --test-force-content-text
cp bench/content bench/content.txt
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 10 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(CONF) test-rewrite
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(CONF) --test-force-content-text test-rewrite
cmp bench/content bench/content.txt
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check -l test.log
$(TESTENV) ./mktest$(EXEEXT) generate 4 6 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 100 sync --test-force-content-text
cp bench/content bench/content.txt
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 100 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-rewrite
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-force-content-text test-rewrite
cmp bench/content bench/content.txt
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 1000 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check -l test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync

30
TODO
View file

@ -16,25 +16,6 @@ The content file is instead missing in all platforms.
- Minor
* Check how to reduce memory occupation using smaller memory block.
We can theoreticallty reduce the memory occupation from 28/36 to
17 bytes. 16 for the hash and 1 for the type.
We can store the parity_position<->file,pos relation using some form of binary tree,
and then drop completely blockarr, and the extra data in memory_block.
A simpler simplification is to convert, blockarr to a parity_position->file,pos
relation, removing the file pointer inside the block.
struct node {
uint32 parity_pos /* parity position */
file* file; /* pointer at the file */
uint32 offset; /* position in the file */
uint32 count; /* number of sequential blocks */
void* left_by_pp; /* tree by parity_pos */
void* right_by_pp;
void* left_by_fp; /* tree by file,offset */
void* right_by_fp;
};
* Restore ownership and permissions, at least in Unix.
* Restore directory timestamps.
@ -96,6 +77,17 @@ rest of the block.
The problem is that we don't have any hash to ensure that the block is partially recovered,
or completely garbage. But it makes sense to anyway write the "most likely" correct one.
* How to handle corrupt copied/moved files ? At now pre-hash checks them,
but if a corruption is found, it stops the sync process.
This is not optimal, as that blocks sync until a manual intervention.
This is the reason why the check of REP (copied) blocks is not enabled by default,
but only inside pre-hash.
A possible solution could be:
If a copied block is corrupt, we can recover the original one from parity (if moved),
or read it directly from disk (if copied), and we can use this one to build the parity,
working around the corruption, that can be later fixed with "fix".
This requires to process first copied blocks, with a possible complex dependency chain.
- Pooling
* Add a new "commit" command to move changes from the pool to the array.

125
cmdline/check.c
View file

@ -56,26 +56,26 @@ struct failed_struct {
int is_outofdate;
unsigned index; /**< Index of the failed block. */
struct snapraid_block* block; /**< The failed block, or BLOCK_DELETED for a deleted block */
struct snapraid_handle* handle; /**< The file containing the failed block, or 0 for a deleted block */
struct snapraid_block* block; /**< The failed block */
struct snapraid_disk* disk; /**< The failed disk. */
struct snapraid_file* file; /**< The failed file. 0 for DELETED block. */
block_off_t file_pos; /**< Offset inside the file */
struct snapraid_handle* handle; /**< The handle containing the failed block, or 0 for a DELETED block */
};
/**
* Checks if a block hash matches the specified buffer.
* Return ==0 if equal
*/
static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero)
static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned pos_size, unsigned char* buffer, unsigned char* buffer_zero)
{
unsigned char hash[HASH_SIZE];
unsigned size;
size = block_file_size(block, state->block_size);
/* now compute the hash of the valid part */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer, size);
memhash(state->prevhash, state->prevhashseed, hash, buffer, pos_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer, size);
memhash(state->hash, state->hashseed, hash, buffer, pos_size);
}
/* compare the hash */
@ -84,8 +84,8 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl
}
/* compare to the end of the block */
if (size < state->block_size) {
if (memcmp(buffer + size, buffer_zero + size, state->block_size - size) != 0) {
if (pos_size < state->block_size) {
if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) {
return -1;
}
}
@ -111,7 +111,8 @@ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned d
&& block_has_updated_hash(failed[failed_map[j]].block)
) {
/* if a hash doesn't match, fail the check */
if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0) {
unsigned pos_size = file_block_size(failed[failed_map[j]].file, failed[failed_map[j]].file_pos, state->block_size);
if (blockcmp(state, rehash, failed[failed_map[j]].block, pos_size, buffer[failed[failed_map[j]].index], buffer_zero) != 0) {
log_tag("hash_error: Hash mismatch on entry %u\n", failed_map[j]);
return 0;
}
@ -188,7 +189,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
for (i = 0; i < failed_count; ++i) {
/* if we are expected to recover this block */
if (!failed[failed_map[i]].is_outofdate
/* if the block has a hash to check */
/* if the block has a hash to check */
&& block_has_updated_hash(failed[failed_map[i]].block)
)
has_hash = 1;
@ -331,14 +332,12 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
else
data = "good";
if (failed[j].handle) {
const char* disk;
const char* sub;
struct snapraid_handle* handle = failed[j].handle;
disk = handle->disk->name;
sub = block_file_get(block)->sub;
if (failed[j].file) {
struct snapraid_disk* disk = failed[j].disk;
struct snapraid_file* file = failed[j].file;
block_off_t file_pos = failed[j].file_pos;
log_tag("entry:%u:%s:%s:%s:%s:%s:%u:\n", j, desc, hash, data, disk, esc(sub), block_file_pos(block));
log_tag("entry:%u:%s:%s:%s:%s:%s:%u:\n", j, desc, hash, data, disk->name, esc(file->sub), file_pos);
} else {
log_tag("entry:%u:%s:%s:%s:\n", j, desc, hash, data);
}
@ -372,7 +371,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP)
/* try to fetch the block using the known hash */
&& (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0
|| state_search_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0)
|| state_search_fetch(state, rehash, failed[j].file, failed[j].file_pos, failed[j].block, buffer[failed[j].index]) == 0)
) {
/* we already have corrected it! */
log_tag("hash_import: Fixed entry %u\n", j);
@ -442,7 +441,8 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
/* instead, if the hash matches, it could be either that the */
/* block after the sync has this hash, or that */
/* we restored the block before the 'sync'. */
if (blockcmp(state, rehash, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) {
unsigned pos_size = file_block_size(failed[j].file, failed[j].file_pos, state->block_size);
if (blockcmp(state, rehash, failed[j].block, pos_size, buffer[failed[j].index], buffer_zero) == 0) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
@ -610,6 +610,7 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
struct snapraid_disk* disk;
struct snapraid_file* collide_file;
struct snapraid_file* file;
block_off_t file_pos;
char path[PATH_MAX];
uint64_t inode;
@ -619,17 +620,17 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s
continue;
}
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* if no file, nothing to do */
continue;
}
file = block_file_get(block);
file = fs_par2file_get(disk, i, &file_pos);
pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub);
/* if it isn't the last block in the file */
if (!block_is_last(block)) {
if (!file_block_is_last(file, file_pos)) {
/* nothing to do */
continue;
}
@ -819,15 +820,16 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct
if (!handle[j].disk)
continue;
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
/* try to recover all files, even the ones without hash */
/* because in some cases we can recover also them */
if (block_has_file(block)
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
break;
if (block_has_file(block)) {
struct snapraid_file* file = fs_par2file_get(handle[j].disk, i, 0);
if (!file_flag_has(file, FILE_IS_EXCLUDED)) { /* only if the file is not filtered out */
one_tocheck = 1;
break;
}
}
}
@ -932,6 +934,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
struct snapraid_disk* disk;
struct snapraid_block* block;
struct snapraid_file* file;
block_off_t file_pos;
unsigned block_state;
/* if the disk position is not used */
@ -943,7 +946,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
}
/* if the disk block is not used */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (block == BLOCK_EMPTY) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
@ -968,17 +971,20 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* store it in the failed set, because potentially */
/* the parity may be still computed with the previous content */
failed[failed_count].is_bad = 0;
failed[failed_count].is_bad = 0; /* note that is_bad==0 <=> file==0 */
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].disk = disk;
failed[failed_count].file = 0;
failed[failed_count].file_pos = 0;
failed[failed_count].handle = 0;
++failed_count;
continue;
}
/* get the file of this block */
file = block_file_get(block);
file = fs_par2file_get(disk, i, &file_pos);
/* if we are only hashing, we can skip excluded files and don't even read them */
if (state->opt.auditonly && file_flag_has(file, FILE_IS_EXCLUDED)) {
@ -1036,10 +1042,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].disk = disk;
failed[failed_count].file = file;
failed[failed_count].file_pos = file_pos;
failed[failed_count].handle = &handle[j];
++failed_count;
log_tag("error:%u:%s:%s: Open error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
log_tag("error:%u:%s:%s: Open error at position %u\n", i, disk->name, esc(file->sub), file_pos);
++error;
continue;
}
@ -1050,9 +1059,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
&& !file_flag_has(file, FILE_IS_EXCLUDED)) {
/* check if the file is changed */
if (handle[j].st.st_size != block_file_get(block)->size
|| handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
if (handle[j].st.st_size != file->size
|| handle[j].st.st_mtime != file->mtime_sec
|| STAT_NSEC(&handle[j].st) != file->mtime_nsec
/* don't check the inode to support filesystem without persistent inodes */
) {
/* report that the file is not synced */
@ -1093,7 +1102,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
}
/* read from the file */
read_size = handle_read(&handle[j], block, buffer[j], state->block_size,
read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size,
log_error, state->opt.expected_missing ? log_expected : 0);
if (read_size == -1) {
/* save the failed block for the check/fix */
@ -1101,10 +1110,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].disk = disk;
failed[failed_count].file = file;
failed[failed_count].file_pos = file_pos;
failed[failed_count].handle = &handle[j];
++failed_count;
log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), file_pos);
++error;
continue;
}
@ -1121,6 +1133,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].disk = disk;
failed[failed_count].file = file;
failed[failed_count].file_pos = file_pos;
failed[failed_count].handle = &handle[j];
++failed_count;
continue;
@ -1144,10 +1159,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].disk = disk;
failed[failed_count].file = file;
failed[failed_count].file_pos = file_pos;
failed[failed_count].handle = &handle[j];
++failed_count;
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), block_file_pos(block), diff);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff);
++error;
continue;
}
@ -1160,6 +1178,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].disk = disk;
failed[failed_count].file = file;
failed[failed_count].file_pos = file_pos;
failed[failed_count].handle = &handle[j];
++failed_count;
continue;
@ -1206,13 +1227,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* print a list of all the errors in files */
for (j = 0; j < failed_count; ++j) {
if (failed[j].is_bad)
log_tag("unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block));
log_tag("unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].disk->name, esc(failed[j].file->sub), failed[j].file_pos);
}
/* keep track of damaged files */
for (j = 0; j < failed_count; ++j) {
if (failed[j].is_bad)
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
file_flag_set(failed[j].file, FILE_IS_DAMAGED);
}
} else {
/* now counts partial recovers */
@ -1224,7 +1245,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
for (j = 0; j < failed_count; ++j) {
if (failed[j].is_bad && failed[j].is_outofdate) {
++partial_recover_error;
log_tag("unrecoverable:%u:%s:%s: Unrecoverable unsynced error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block));
log_tag("unrecoverable:%u:%s:%s: Unrecoverable unsynced error at position %u\n", i, failed[j].disk->name, esc(failed[j].file->sub), failed[j].file_pos);
}
}
if (partial_recover_error != 0) {
@ -1258,15 +1279,15 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
continue;
/* do not fix if the file is excluded */
if (file_flag_has(block_file_get(failed[j].block), FILE_IS_EXCLUDED)
|| (state->opt.syncedonly && file_flag_has(block_file_get(failed[j].block), FILE_IS_UNSYNCED)))
if (file_flag_has(failed[j].file, FILE_IS_EXCLUDED)
|| (state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED)))
continue;
ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size);
ret = handle_write(failed[j].handle, failed[j].file_pos, buffer[failed[j].index], state->block_size);
if (ret == -1) {
/* LCOV_EXCL_START */
/* mark the file as damaged */
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
file_flag_set(failed[j].file, FILE_IS_DAMAGED);
if (errno == EACCES) {
log_fatal("WARNING! Please give write permission to the file.\n");
@ -1283,15 +1304,15 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* if we are not sure that the recovered content is uptodate */
if (failed[j].is_outofdate) {
/* mark the file as damaged */
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
file_flag_set(failed[j].file, FILE_IS_DAMAGED);
continue;
}
/* mark the file as containing some fixes */
/* note that it could be also marked as damaged in other iterations */
file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED);
file_flag_set(failed[j].file, FILE_IS_FIXED);
log_tag("fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block));
log_tag("fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].disk->name, esc(failed[j].file->sub), failed[j].file_pos);
++recovered_error;
}
@ -1323,7 +1344,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* meaning that we could fix this file if we try */
for (j = 0; j < failed_count; ++j) {
if (failed[j].is_bad) {
file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED);
file_flag_set(failed[j].file, FILE_IS_FIXED);
}
}
}
@ -1333,7 +1354,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* to report that the file is damaged, and we don't know if we can fix it */
for (j = 0; j < failed_count; ++j) {
if (failed[j].is_bad) {
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
file_flag_set(failed[j].file, FILE_IS_DAMAGED);
}
}
}

11
cmdline/dry.c
View file

@ -58,16 +58,17 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
/* for each disk, process the block */
for (j = 0; j < diskmax; ++j) {
int read_size;
struct snapraid_block* block = BLOCK_EMPTY;
struct snapraid_block* block;
struct snapraid_disk* disk = handle[j].disk;
struct snapraid_file* file;
block_off_t file_pos;
if (!disk) {
/* if no disk, nothing to do */
continue;
}
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* if no file, nothing to do */
@ -75,7 +76,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
}
/* get the file of this block */
file = block_file_get(block);
file = fs_par2file_get(disk, i, &file_pos);
/* until now is CPU */
state_usage_cpu(state);
@ -108,9 +109,9 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
}
/* read from the file */
read_size = handle_read(&handle[j], block, buffer_aligned, state->block_size, log_error, 0);
read_size = handle_read(&handle[j], file_pos, buffer_aligned, state->block_size, log_error, 0);
if (read_size == -1) {
log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block));
log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), file_pos);
++error;
continue;
}

6
cmdline/dup.c
View file

@ -50,9 +50,11 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE);
struct snapraid_block* block = fs_file2block_get(file, i);
if (!block_has_updated_hash(&file->blockvec[i])) {
memcpy(buf + i * HASH_SIZE, block->hash, HASH_SIZE);
if (!block_has_updated_hash(block)) {
free(buf);
free(hash);
return 0;

706
cmdline/elem.c
View file

@ -309,7 +309,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub)
return 1;
}
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file)
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file)
{
unsigned i;
@ -318,12 +318,8 @@ int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapr
/* check each block of the file */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block;
snapraid_info info;
block = &file->blockvec[i];
info = info_get(infoarr, block->parity_pos);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
snapraid_info info = info_get(infoarr, parity_pos);
/* if the file has a bad block, don't exclude it */
if (info_get_bad(info))
@ -359,75 +355,6 @@ int filter_content(tommy_list* contentlist, const char* path)
return 0;
}
block_off_t block_file_pos(struct snapraid_block* block)
{
struct snapraid_file* file = block_file_get(block);
if (block < file->blockvec || block >= file->blockvec + file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in block %u ownership\n", block->parity_pos);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
return block - file->blockvec;
}
int block_is_last(struct snapraid_block* block)
{
struct snapraid_file* file = block_file_get(block);
return block == file->blockvec + file->blockmax - 1;
}
unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
{
block_off_t pos = block_file_pos(block);
/* if it's the last block */
if (pos + 1 == block_file_get(block)->blockmax) {
unsigned remainder;
if (block_file_get(block)->size == 0)
return 0;
remainder = block_file_get(block)->size % block_size;
if (remainder == 0)
remainder = block_size;
return remainder;
}
return block_size;
}
struct snapraid_deleted* deleted_alloc(void)
{
struct snapraid_deleted* deleted;
deleted = malloc_nofail(sizeof(struct snapraid_deleted));
/* set the state as deleted */
deleted->block.file_mixed = BLOCK_STATE_DELETED;
return deleted;
}
struct snapraid_deleted* deleted_dup(struct snapraid_block* block)
{
struct snapraid_deleted* deleted;
deleted = deleted_alloc();
/* copy data from deleted block */
deleted->block.parity_pos = block->parity_pos;
memcpy(deleted->block.hash, block->hash, HASH_SIZE);
return deleted;
}
void deleted_free(struct snapraid_deleted* deleted)
{
free(deleted);
}
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical)
{
struct snapraid_file* file;
@ -444,15 +371,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
file->flag = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].parity_pos = POS_INVALID;
file->blockvec[i].file_mixed = 0;
/* set the file */
block_file_set(&file->blockvec[i], file);
/* set an invalid hash */
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
hash_invalid_set(file->blockvec[i].hash);
}
@ -460,10 +379,36 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
return file;
}
struct snapraid_file* file_dup(struct snapraid_file* copy)
{
struct snapraid_file* file;
block_off_t i;
file = malloc_nofail(sizeof(struct snapraid_file));
file->sub = strdup_nofail(copy->sub);
file->size = copy->size;
file->blockmax = copy->blockmax;
file->mtime_sec = copy->mtime_sec;
file->mtime_nsec = copy->mtime_nsec;
file->inode = copy->inode;
file->physical = copy->physical;
file->flag = copy->flag;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
for (i = 0; i < file->blockmax; ++i) {
file->blockvec[i].state = copy->blockvec[i].state;
memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE);
}
return file;
}
void file_free(struct snapraid_file* file)
{
free(file->sub);
file->sub = 0;
free(file->blockvec);
file->blockvec = 0;
free(file);
}
@ -520,6 +465,37 @@ const char* file_name(const struct snapraid_file* file)
return r;
}
unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size)
{
/* if it's the last block */
if (file_pos + 1 == file->blockmax) {
unsigned remainder;
if (file->size == 0)
return 0;
remainder = file->size % block_size;
if (remainder == 0)
remainder = block_size;
return remainder;
}
return block_size;
}
int file_block_is_last(struct snapraid_file* file, block_off_t file_pos)
{
if (file_pos == 0 && file->blockmax == 0)
return 1;
if (file_pos >= file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file block position\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
return file_pos == file->blockmax - 1;
}
int file_inode_compare_to_arg(const void* void_arg, const void* void_data)
{
const uint64_t* arg = void_arg;
@ -627,6 +603,55 @@ int file_pathstamp_compare(const void* void_a, const void* void_b)
return file_stamp_compare(void_a, void_b);
}
struct snapraid_chunk* chunk_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count)
{
struct snapraid_chunk* chunk;
chunk = malloc_nofail(sizeof(struct snapraid_chunk));
chunk->parity_pos = parity_pos;
chunk->file = file;
chunk->file_pos = file_pos;
chunk->count = count;
return chunk;
}
void chunk_free(struct snapraid_chunk* chunk)
{
free(chunk);
}
int chunk_parity_compare(const void* void_a, const void* void_b)
{
const struct snapraid_chunk* arg_a = void_a;
const struct snapraid_chunk* arg_b = void_b;
if (arg_a->parity_pos < arg_b->parity_pos)
return -1;
if (arg_a->parity_pos > arg_b->parity_pos)
return 1;
return 0;
}
int chunk_file_compare(const void* void_a, const void* void_b)
{
const struct snapraid_chunk* arg_a = void_a;
const struct snapraid_chunk* arg_b = void_b;
if (arg_a->file < arg_b->file)
return -1;
if (arg_a->file > arg_b->file)
return 1;
if (arg_a->file_pos < arg_b->file_pos)
return -1;
if (arg_a->file_pos > arg_b->file_pos)
return 1;
return 0;
}
struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag)
{
struct snapraid_link* link;
@ -719,7 +744,9 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
tommy_hashdyn_init(&disk->linkset);
tommy_list_init(&disk->dirlist);
tommy_hashdyn_init(&disk->dirset);
tommy_arrayblk_init(&disk->blockarr);
tommy_tree_init(&disk->fs_parity, chunk_parity_compare);
tommy_tree_init(&disk->fs_file, chunk_file_compare);
disk->fs_last = 0;
return disk;
}
@ -727,7 +754,8 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
void disk_free(struct snapraid_disk* disk)
{
tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free);
tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)deleted_free);
tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)file_free);
tommy_tree_foreach(&disk->fs_file, (tommy_foreach_func*)chunk_free);
tommy_hashdyn_done(&disk->inodeset);
tommy_hashdyn_done(&disk->pathset);
tommy_hashdyn_done(&disk->stampset);
@ -735,13 +763,32 @@ void disk_free(struct snapraid_disk* disk)
tommy_hashdyn_done(&disk->linkset);
tommy_list_foreach(&disk->dirlist, (tommy_foreach_func*)dir_free);
tommy_hashdyn_done(&disk->dirset);
tommy_arrayblk_done(&disk->blockarr);
free(disk);
}
struct chunk_disk_empty {
block_off_t blockmax;
};
/**
* Compare the chunk if inside the specified blockmax.
*/
static int chunk_disk_empty_compare(const void* void_a, const void* void_b)
{
const struct chunk_disk_empty* arg_a = void_a;
const struct snapraid_chunk* arg_b = void_b;
/* if the block is inside the specified blockmax, it's found */
if (arg_a->blockmax > arg_b->parity_pos)
return 0;
/* otherwise search for a smaller one */
return -1;
}
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax)
{
block_off_t i;
struct chunk_disk_empty arg = { blockmax };
/* if there is an element, it's not empty */
/* even if links and dirs have no block allocation */
@ -752,35 +799,470 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax)
if (!tommy_list_empty(&disk->dirlist))
return 0;
/* limit the search to the size of the disk */
if (blockmax > tommy_arrayblk_size(&disk->blockarr))
blockmax = tommy_arrayblk_size(&disk->blockarr);
/* checks all the blocks to search for deleted ones */
/* this search is slow, but it's done only if no file is present */
for (i = 0; i < blockmax; ++i) {
struct snapraid_block* block = tommy_arrayblk_get(&disk->blockarr, i);
unsigned block_state = block_state_get(block);
switch (block_state) {
case BLOCK_STATE_EMPTY :
/* empty block are expected for an empty disk */
break;
case BLOCK_STATE_DELETED :
/* if there is a deleted block, the disk is not empty */
return 0;
default :
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for used block in disk '%s' without files\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* search for any chunk inside blockmax */
if (tommy_tree_search_compare(&disk->fs_parity, chunk_disk_empty_compare, &arg) != 0)
return 0;
/* finally, it's empty */
return 1;
}
struct chunk_disk_size {
block_off_t size;
};
/**
* Compare the chunk by highest parity position.
*
* The maximum parity position is stored as size.
*/
static int chunk_disk_size_compare(const void* void_a, const void* void_b)
{
struct chunk_disk_size* arg_a = (void*)void_a;
const struct snapraid_chunk* arg_b = void_b;
/* get the maximum size */
if (arg_a->size < arg_b->parity_pos + arg_b->count)
arg_a->size = arg_b->parity_pos + arg_b->count;
/* search always for a bigger one */
return 1;
}
block_off_t disk_size(struct snapraid_disk* disk)
{
struct chunk_disk_size arg = { 0 };
tommy_tree_search_compare(&disk->fs_parity, chunk_disk_size_compare, &arg);
return arg.size;
}
struct chunk_check {
const struct snapraid_chunk* prev;
int result;
};
void chunk_parity_check_foreach(void* void_arg, void* void_obj)
{
struct chunk_check* arg = void_arg;
const struct snapraid_chunk* obj = void_obj;
const struct snapraid_chunk* prev = arg->prev;
/* set the next previous block */
arg->prev = obj;
/* stop reporting if too many errors */
if (arg->result > 100) {
/* LCOV_EXCL_START */
return;
/* LCOV_EXCL_STOP */
}
/* check only if there is a previous block */
if (!prev)
return;
/* check the order */
if (prev->parity_pos >= obj->parity_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in parity order for files '%s' at '%u:%u' and '%s' at '%u:%u'\n",
prev->file->sub, prev->parity_pos, prev->count, obj->file->sub, obj->parity_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
/* check that the chunks don't overlap */
if (prev->parity_pos + prev->count > obj->parity_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for parity overlap for files '%s' at '%u:%u' and '%s' at '%u:%u'\n",
prev->file->sub, prev->parity_pos, prev->count, obj->file->sub, obj->parity_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
void chunk_file_check_foreach(void* void_arg, void* void_obj)
{
struct chunk_check* arg = void_arg;
const struct snapraid_chunk* obj = void_obj;
const struct snapraid_chunk* prev = arg->prev;
/* set the next previous block */
arg->prev = obj;
/* stop reporting if too many errors */
if (arg->result > 100) {
/* LCOV_EXCL_START */
return;
/* LCOV_EXCL_STOP */
}
/* note that for deleted files, some chunks may be missing */
/* if the files are different */
if (!prev || prev->file != obj->file) {
if (prev != 0) {
if (file_flag_has(prev->file, FILE_IS_DELETED)) {
/* check that the chunk doesn't overflow the file */
if (prev->file_pos + prev->count > prev->file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in delete end for file '%s' at '%u:%u' overflowing size '%u'\n",
prev->file->sub, prev->file_pos, prev->count, prev->file->blockmax);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
} else {
/* check that the chunk ends the file */
if (prev->file_pos + prev->count != prev->file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file end for file '%s' at '%u:%u' instead of size '%u'\n",
prev->file->sub, prev->file_pos, prev->count, prev->file->blockmax);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
}
if (file_flag_has(obj->file, FILE_IS_DELETED)) {
/* check that the chunk doesn't overflow the file */
if (obj->file_pos + obj->count > obj->file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in delete start for file '%s' at '%u:%u' overflowing size '%u'\n",
obj->file->sub, obj->file_pos, obj->count, obj->file->blockmax);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
} else {
/* check that the chunk starts the file */
if (obj->file_pos != 0) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file start for file '%s' at '%u:%u'\n",
obj->file->sub, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
} else {
/* check the order */
if (prev->file_pos >= obj->file_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file order for file '%s' at '%u:%u' and at '%u:%u'\n",
prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
if (file_flag_has(obj->file, FILE_IS_DELETED)) {
/* check that the chunks don't overlap */
if (prev->file_pos + prev->count > obj->file_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in delete sequence for file '%s' at '%u:%u' and at '%u:%u'\n",
prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
} else {
/* check that the chunks are sequential */
if (prev->file_pos + prev->count != obj->file_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file sequence for file '%s' at '%u:%u' and at '%u:%u'\n",
prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
}
}
int fs_check(struct snapraid_disk* disk)
{
struct chunk_check arg;
/* error count starts from 0 */
arg.result = 0;
/* check parity sequence */
arg.prev = 0;
tommy_tree_foreach_arg(&disk->fs_parity, chunk_parity_check_foreach, &arg);
/* check file sequence */
arg.prev = 0;
tommy_tree_foreach_arg(&disk->fs_file, chunk_file_check_foreach, &arg);
if (arg.result != 0)
return -1;
return 0;
}
struct chunk_parity_inside {
block_off_t parity_pos;
};
/**
* Compare the chunk if containing the specified parity position.
*/
static int chunk_parity_inside_compare(const void* void_a, const void* void_b)
{
const struct chunk_parity_inside* arg_a = void_a;
const struct snapraid_chunk* arg_b = void_b;
if (arg_a->parity_pos < arg_b->parity_pos)
return -1;
if (arg_a->parity_pos >= arg_b->parity_pos + arg_b->count)
return 1;
return 0;
}
/**
* Seach the chunk at the specified parity position.
* The search is optimized for sequential accesses.
* \return If not found return 0
*/
static struct snapraid_chunk* fs_par2chunk_get(struct snapraid_disk* disk, block_off_t parity_pos)
{
struct snapraid_chunk* chunk;
/* check if the last accessed chunk matches */
if (disk->fs_last
&& parity_pos >= disk->fs_last->parity_pos
&& parity_pos < disk->fs_last->parity_pos + disk->fs_last->count
) {
chunk = disk->fs_last;
} else {
struct chunk_parity_inside arg = { parity_pos };
chunk = tommy_tree_search_compare(&disk->fs_parity, chunk_parity_inside_compare, &arg);
}
if (!chunk)
return 0;
/* store the last accessed chunk */
disk->fs_last = chunk;
return chunk;
}
struct chunk_file_inside {
struct snapraid_file* file;
block_off_t file_pos;
};
/**
* Compare the chunk if containing the specified file position.
*/
static int chunk_file_inside_compare(const void* void_a, const void* void_b)
{
const struct chunk_file_inside* arg_a = void_a;
const struct snapraid_chunk* arg_b = void_b;
if (arg_a->file < arg_b->file)
return -1;
if (arg_a->file > arg_b->file)
return 1;
if (arg_a->file_pos < arg_b->file_pos)
return -1;
if (arg_a->file_pos >= arg_b->file_pos + arg_b->count)
return 1;
return 0;
}
/**
* Seach the chunk at the specified file position.
* The search is optimized for sequential accesses.
* \return If not found return 0
*/
static struct snapraid_chunk* fs_file2chunk_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
struct snapraid_chunk* chunk;
/* check if the last accessed chunk matches */
if (disk->fs_last
&& file == disk->fs_last->file
&& file_pos >= disk->fs_last->file_pos
&& file_pos < disk->fs_last->file_pos + disk->fs_last->count
) {
chunk = disk->fs_last;
} else {
struct chunk_file_inside arg = { file, file_pos };
chunk = tommy_tree_search_compare(&disk->fs_file, chunk_file_inside_compare, &arg);
}
if (!chunk)
return 0;
/* store the last accessed chunk */
disk->fs_last = chunk;
return chunk;
}
struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos)
{
struct snapraid_chunk* chunk;
chunk = fs_par2chunk_get(disk, parity_pos);
if (!chunk)
return 0;
if (file_pos)
*file_pos = chunk->file_pos + (parity_pos - chunk->parity_pos);
return chunk->file;
}
block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
struct snapraid_chunk* chunk;
chunk = fs_file2chunk_get(disk, file, file_pos);
if (!chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency when resolving a file to a not existing chunk in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
return chunk->parity_pos + (file_pos - chunk->file_pos);
}
void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos)
{
struct snapraid_chunk* chunk;
struct snapraid_chunk* parity_chunk;
struct snapraid_chunk* file_chunk;
if (file_pos > 0) {
/* search an existing chunk for the previous file_pos */
chunk = fs_file2chunk_get(disk, file, file_pos - 1);
if (chunk != 0 && parity_pos == chunk->parity_pos + chunk->count) {
/* ensure that we are extending the chunk at the end */
if (file_pos != chunk->file_pos + chunk->count) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency extending a chunk in the middle in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* extend the existing chunk */
++chunk->count;
return;
}
}
/* a chunk doesn't exist, and we have to create a new one */
chunk = chunk_alloc(parity_pos, file, file_pos, 1);
/* insert the chunk in the trees */
parity_chunk = tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk);
file_chunk = tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk);
if (parity_chunk != chunk || file_chunk != chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency when allocating a chunk that already exists in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* store the last accessed chunk */
disk->fs_last = chunk;
}
void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos)
{
struct snapraid_chunk* chunk;
struct snapraid_chunk* second_chunk;
struct snapraid_chunk* parity_chunk;
struct snapraid_chunk* file_chunk;
block_off_t first_count, second_count;
chunk = fs_par2chunk_get(disk, parity_pos);
if (!chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency when deallocating a not existing chunk in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* if it's the only block of the chunk, delete it */
if (chunk->count == 1) {
/* remove from the trees */
tommy_tree_remove(&disk->fs_parity, chunk);
tommy_tree_remove(&disk->fs_file, chunk);
/* deallocate */
chunk_free(chunk);
/* clear the last accessed chunk */
disk->fs_last = 0;
return;
}
/* if it's at the start of the chunk, shrink the chunk */
if (parity_pos == chunk->parity_pos) {
++chunk->parity_pos;
++chunk->file_pos;
--chunk->count;
return;
}
/* if it's at the end of the chunk, shrink the chunk */
if (parity_pos == chunk->parity_pos + chunk->count - 1) {
--chunk->count;
return;
}
/* otherwise it's in the middle */
first_count = parity_pos - chunk->parity_pos;
second_count = chunk->count - first_count - 1;
/* adjust the first chunk */
chunk->count = first_count;
/* allocate the second chunk */
second_chunk = chunk_alloc(chunk->parity_pos + first_count + 1, chunk->file, chunk->file_pos + first_count + 1, second_count);
/* insert the chunk in the trees */
parity_chunk = tommy_tree_insert(&disk->fs_parity, &second_chunk->parity_node, second_chunk);
file_chunk = tommy_tree_insert(&disk->fs_file, &second_chunk->file_node, second_chunk);
if (parity_chunk != second_chunk || file_chunk != second_chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency when splitting a chunk in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* store the last accessed chunk */
disk->fs_last = second_chunk;
}
struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos)
{
struct snapraid_file* file;
block_off_t file_pos;
file = fs_par2file_get(disk, parity_pos, &file_pos);
if (!file)
return BLOCK_EMPTY;
return fs_file2block_get(file, file_pos);
}
struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid)
{
struct snapraid_map* map;

343
cmdline/elem.h
View file

@ -20,8 +20,9 @@
#include "util.h"
#include "support.h"
#include "tommyds/tommylist.h"
#include "tommyds/tommyhash.h"
#include "tommyds/tommylist.h"
#include "tommyds/tommytree.h"
#include "tommyds/tommyhashdyn.h"
#include "tommyds/tommyarray.h"
#include "tommyds/tommyarrayblk.h"
@ -68,6 +69,17 @@ struct snapraid_filter {
tommy_node node; /**< Next node in the list. */
};
/**
* Block pointer used to represent unused blocks.
*/
#define BLOCK_EMPTY 0
/**
* This block is an empty one.
* Note that an empty block is represent with ::BLOCK_EMPTY.
*/
#define BLOCK_STATE_EMPTY 0
/**
* The block has both the hash and the parity computed.
* This is the normal state of a saved block.
@ -141,33 +153,14 @@ struct snapraid_filter {
*/
#define BLOCK_STATE_DELETED 4
/**
* Mask used to store the previous states of the block.
* Used to mix them inside the file pointer.
*/
#define BLOCK_STATE_MASK 7
/**
* This block is an empty one.
* Note that this state cannot be stored inside the block,
* and it's represented by the block ::BLOCK_EMPTY.
*/
#define BLOCK_STATE_EMPTY 8
/**
* Block of a file.
*/
struct snapraid_block {
uintptr_t file_mixed; /**< Back pointer to the file owning this block, mixed with some flags. */
block_off_t parity_pos; /**< Position of the block in the parity. */
unsigned char hash[HASH_SIZE]; /**< Hash of the block. */
unsigned char state; /**< State of the block. */
};
/**
* Block pointer used to mark unused blocks.
*/
#define BLOCK_EMPTY 0
/**
* If a file is present in the disk.
* It's used only in scan to detect present and missing files.
@ -236,6 +229,16 @@ struct snapraid_block {
*/
#define FILE_IS_WITHOUT_INODE 0x200
/**
* The file is deleted.
* This happens when a file is deleted from the array,
* but it's keep inside the parity until the next sync.
*
* During the filesystem check we needs this information,
* because deleted files may be present only partially.
*/
#define FILE_IS_DELETED 0x400
#define FILE_IS_HARDLINK 0x1000 /**< If it's an hardlink. */
#define FILE_IS_SYMLINK 0x2000 /**< If it's a file symlink. */
#define FILE_IS_SYMDIR 0x4000 /**< If it's a dir symlink for Windows. Not yet supported. */
@ -289,18 +292,17 @@ struct snapraid_dir {
};
/**
* Deleted entry.
* Chunk.
*
* A chunk represents a fragment of a file mapped into the parity.
*/
struct snapraid_deleted {
/**
* Deleted block.
* This block is always in state BLOCK_STATE_DELETED,
* and it's used to keep the old hash during a sync.
*/
struct snapraid_block block;
/* nodes for data structures */
tommy_node node;
struct snapraid_chunk {
struct snapraid_file* file; /* File containing this chunk. */
block_off_t parity_pos; /**< Parity position. */
block_off_t file_pos; /**< Position in the file. */
block_off_t count; /**< Number of sequential blocks in the file and parity. */
tommy_tree_node parity_node; /**< Tree sorted by <parity_pos>. */
tommy_tree_node file_node; /**< Tree sorter by <file,file_pos>. */
};
/**
@ -331,12 +333,18 @@ struct snapraid_disk {
int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
/**<
* Block array of the disk.
*
* Each element points to a snapraid_block structure, or it's BLOCK_EMPTY.
/**
* Mapping of chunks in the parity.
* Sorted by <parity_pos> and by <file,file_pos>
*/
tommy_arrayblk blockarr;
tommy_tree fs_parity;
tommy_tree fs_file;
/**
* Last chunk we accessed.
* It's used to optimize access of sequential blocks.
*/
struct snapraid_chunk* fs_last;
/**
* List of all the snapraid_file for the disk.
@ -344,7 +352,7 @@ struct snapraid_disk {
tommy_list filelist;
/**
* List of all the snapraid_deleted blocks for the disk.
* List of all the deleted file for the disk.
*
* These files are kept allocated, because the blocks are still referenced in
* the ::blockarr.
@ -465,7 +473,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub);
* Filter a file if bad.
* Return !=0 if the file is correct and it should be excluded.
*/
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file);
int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file);
/**
* Filters a dir using a list of filters.
@ -480,85 +488,6 @@ int filter_dir(tommy_list* filterlist, struct snapraid_filter** reason, const ch
*/
int filter_content(tommy_list* contentlist, const char* path);
/**
* Gets the file containing the block.
*/
static inline struct snapraid_file* block_file_get(struct snapraid_block* block)
{
return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK);
}
/**
* Sets the file containing the block.
*/
static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file)
{
uintptr_t ptr = (uintptr_t)file;
/* ensure that the pointer doesn't use the flag space */
if ((ptr & (uintptr_t)BLOCK_STATE_MASK) != 0) {
/* LCOV_EXCL_START */
log_fatal("Internal error for pointer not aligned\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
block->file_mixed = (block->file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr;
}
/**
* Get the state of the block.
*
* For this function, it's allowed to pass a NULL block
* pointer than results in the BLOCK_STATE_EMPTY state.
*/
static inline unsigned block_state_get(const struct snapraid_block* block)
{
if (block == BLOCK_EMPTY)
return BLOCK_STATE_EMPTY;
return block->file_mixed & BLOCK_STATE_MASK;
}
static inline void block_state_set(struct snapraid_block* block, unsigned state)
{
/* ensure that the state can be stored inside the file pointer */
if ((state & BLOCK_STATE_MASK) != state) {
/* LCOV_EXCL_START */
log_fatal("Internal error when setting the block state %u\n", state);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
block->file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK;
block->file_mixed |= state & BLOCK_STATE_MASK;
}
/**
* Checks if the specified block has an updated hash.
*
* Note that EMPTY / CHG / DELETED return 0.
*/
static inline int block_has_updated_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
}
/**
* Checks if the specified block has a past hash,
* i.e. the hash of the data that it's now overwritten or lost.
*
* Note that EMPTY / BLK / REP return 0.
*/
static inline int block_has_past_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
}
/**
* Checks if the specified hash is invalid.
*
@ -615,6 +544,53 @@ static inline void hash_zero_set(unsigned char* hash)
memset(hash, 0xFF, HASH_SIZE);
}
/**
* Get the state of the block.
*
* For this function, it's allowed to pass a NULL block
* pointer than results in the BLOCK_STATE_EMPTY state.
*/
static inline unsigned block_state_get(const struct snapraid_block* block)
{
if (block == BLOCK_EMPTY)
return BLOCK_STATE_EMPTY;
return block->state;
}
/**
* Set the state of the block.
*/
static inline void block_state_set(struct snapraid_block* block, unsigned state)
{
block->state = state;
}
/**
* Checks if the specified block has an updated hash.
*
* Note that EMPTY / CHG / DELETED return 0.
*/
static inline int block_has_updated_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
}
/**
* Checks if the specified block has a past hash,
* i.e. the hash of the data that it's now overwritten or lost.
*
* Note that EMPTY / BLK / REP return 0.
*/
static inline int block_has_past_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
}
/**
* Checks if the specified block is part of a file.
*
@ -653,37 +629,6 @@ static inline int block_has_file_and_valid_parity(const struct snapraid_block* b
return state == BLOCK_STATE_BLK;
}
/**
* Gets the relative position of a block inside the file.
*/
block_off_t block_file_pos(struct snapraid_block* block);
/**
* Checks if the block is the last in the file.
*/
int block_is_last(struct snapraid_block* block);
/**
* Gets the size in bytes of the block.
* If it's the last block of a file it could be less than block_size.
*/
unsigned block_file_size(struct snapraid_block* block, unsigned block_size);
/**
* Allocates a deleted block.
*/
struct snapraid_deleted* deleted_alloc(void);
/**
* Allocates a deleted block from a real one.
*/
struct snapraid_deleted* deleted_dup(struct snapraid_block* block);
/**
* Frees a deleted block.
*/
void deleted_free(struct snapraid_deleted* deleted);
static inline int file_flag_has(const struct snapraid_file* file, unsigned mask)
{
return (file->flag & mask) == mask;
@ -700,12 +645,17 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
}
/**
* Allocates a file.
* Allocate a file.
*/
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
/**
* Deallocates a file.
* Duplicate a file.
*/
struct snapraid_file* file_dup(struct snapraid_file* copy);
/**
* Deallocate a file.
*/
void file_free(struct snapraid_file* file);
@ -724,6 +674,17 @@ void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file);
*/
const char* file_name(const struct snapraid_file* file);
/**
* Checks if the block is the last in the file.
*/
int file_block_is_last(struct snapraid_file* file, block_off_t file_pos);
/**
* Gets the size in bytes of the block.
* If it's the last block of a file it could be less than block_size.
*/
unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size);
/**
* Compares a file with an inode.
*/
@ -781,6 +742,26 @@ static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec,
return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec)));
}
/**
* Allocate a chunk.
*/
struct snapraid_chunk* chunk_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count);
/**
* Deallocate a chunk.
*/
void chunk_free(struct snapraid_chunk* chunk);
/**
* Compare chunk by parity position.
*/
int chunk_parity_compare(const void* void_a, const void* void_b);
/**
* Compare chunk by file and file position.
*/
int chunk_file_compare(const void* void_a, const void* void_b);
static inline int link_flag_has(const struct snapraid_link* link, unsigned mask)
{
return (link->flag & mask) == mask;
@ -884,16 +865,9 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
void disk_free(struct snapraid_disk* disk);
/**
* Gets a specific block of a disk.
* Returns 0 if the block is over the end of the disk or not used.
* Get the size of the disk in blocks.
*/
static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, block_off_t pos)
{
if (pos < tommy_arrayblk_size(&disk->blockarr))
return tommy_arrayblk_get(&disk->blockarr, pos);
else
return BLOCK_EMPTY;
}
block_off_t disk_size(struct snapraid_disk* disk);
/**
* Check if a disk is totally empty and can be discarded from the content file.
@ -903,6 +877,55 @@ static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk,
*/
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
/**
* Check the filesystem for errors.
* Return 0 if it's OK.
*/
int fs_check(struct snapraid_disk* disk);
/**
* Allocate a parity position for the specified file position.
*
* After this call you can use the par2file/par2block operations
* to query the relation.
*/
void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos);
/**
* Deallocate the parity position.
*
* After this call the par2file/par2block operations
* won't find anymore the parity association.
*/
void fs_deallocate(struct snapraid_disk* disk, block_off_t pos);
/**
* Get the block from the file position.
*/
static inline struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos)
{
assert(file_pos < file->blockmax);
return &file->blockvec[file_pos];
}
/**
* Get the file position from the parity position.
* Return 0 if no file is using it.
*/
struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos);
/**
* Get the parity position from the file position.
*/
block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos);
/**
* Get the block from the parity position.
* Returns BLOCK_EMPTY==0 if the block is over the end of the disk or not used.
*/
struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos);
/**
* Allocates a disk mapping.
* Uses uuid="" if not available.

12
cmdline/handle.c
View file

@ -248,14 +248,14 @@ int handle_close(struct snapraid_handle* handle)
return 0;
}
int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing)
int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing)
{
ssize_t read_ret;
data_off_t offset;
unsigned read_size;
unsigned count;
offset = block_file_pos(block) * (data_off_t)block_size;
offset = file_pos * (data_off_t)block_size;
if (!out_missing)
out_missing = out;
@ -270,7 +270,7 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un
return -1;
}
read_size = block_file_size(block, block_size);
read_size = file_block_size(handle->file, file_pos, block_size);
#if !HAVE_PREAD
if (lseek(handle->f, offset, SEEK_SET) != offset) {
@ -323,15 +323,15 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un
return read_size;
}
int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size)
int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size)
{
ssize_t write_ret;
data_off_t offset;
unsigned write_size;
offset = block_file_pos(block) * (data_off_t)block_size;
offset = file_pos * (data_off_t)block_size;
write_size = block_file_size(block, block_size);
write_size = file_block_size(handle->file, file_pos, block_size);
#if HAVE_PWRITE
write_ret = pwrite(handle->f, block_buffer, write_size, offset);

4
cmdline/handle.h
View file

@ -64,12 +64,12 @@ int handle_close(struct snapraid_handle* handle);
* Read a block from a file.
* If the read block is shorter, it's padded with 0.
*/
int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing);
int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing);
/**
* Writes a block to a file.
*/
int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size);
int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size);
/**
* Changes the modification time of the file to the saved value.

6
cmdline/import.c
View file

@ -66,7 +66,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
file->path = strdup_nofail(path);
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block));
buffer = malloc_nofail(block_size);
@ -98,7 +98,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
offset = 0;
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_import_block* block = &file->blockvec[i];
struct snapraid_import_block* block = &file->blockimp[i];
unsigned read_size = block_size;
if (read_size > size)
read_size = size;
@ -145,7 +145,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
void import_file_free(struct snapraid_import_file* file)
{
free(file->path);
free(file->blockvec);
free(file->blockimp);
free(file);
}

2
cmdline/import.h
View file

@ -46,7 +46,7 @@ struct snapraid_import_block {
*/
struct snapraid_import_file {
data_off_t size; /**< Size of the file. */
struct snapraid_import_block* blockvec; /**< All the blocks of the file. */
struct snapraid_import_block* blockimp; /**< All the blocks of the file. */
block_off_t blockmax; /**< Number of blocks. */
char* path; /**< Full path of the file. */

13
cmdline/parity.c
View file

@ -37,12 +37,12 @@ block_off_t parity_allocated_size(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
block_off_t block = disk_size(disk);
/* decrease the block until an allocated one, but part of a file */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file(fs_par2block_get(disk, block - 1)))
--block;
/* get the highest value */
@ -64,10 +64,10 @@ block_off_t parity_used_size(struct snapraid_state* state)
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
block_off_t block = disk_size(disk);
/* decrease the block until an used one */
while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file_and_valid_parity(fs_par2block_get(disk, block - 1)))
--block;
/* get the highest value */
@ -95,7 +95,7 @@ int parity_is_invalid(struct snapraid_state* state)
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
struct snapraid_block* block = fs_par2block_get(disk, i);
if (block_has_file(block))
one_valid = 1;
@ -134,7 +134,8 @@ void parity_overflow(struct snapraid_state* state, data_off_t size)
struct snapraid_file* file = j->data;
if (file->blockmax > 0) {
if (file->blockvec[file->blockmax - 1].parity_pos >= blockalloc) {
block_off_t parity_pos = fs_file2par_get(disk, file, file->blockmax - 1);
if (parity_pos >= blockalloc) {
log_tag("outofparity:%s:%s\n", disk->name, esc(file->sub));
if (first) {
first = 0;

454
cmdline/scan.c
View file

@ -155,34 +155,128 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub,
}
/**
* Removes the specified file from the parity.
* Insert the specified file in the parity.
*/
static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t parity_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
parity_pos = disk->first_free_block;
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block;
struct snapraid_block* over_block;
snapraid_info info;
/* increment the position until the first really free block */
while (block_has_file(fs_par2block_get(disk, parity_pos)))
++parity_pos;
/* get block we are going to overwrite, if any */
over_block = fs_par2block_get(disk, parity_pos);
/* deallocate it */
if (over_block != BLOCK_EMPTY)
fs_deallocate(disk, parity_pos);
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* get the new block we are going to write */
block = fs_file2block_get(file, i);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(block, BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
unsigned over_state;
/* convert to a CHG block */
block_state_set(block, BLOCK_STATE_CHG);
/* state of the block we are going to overwrite */
over_state = block_state_get(over_block);
/* if the block is an empty one */
if (over_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(over_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the past hash */
if (!state->clear_past_hash) {
/* in this case we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
fs_allocate(disk, parity_pos, file, i);
/* set the new free position */
disk->first_free_block = parity_pos + 1;
}
/* insert in the list of contained files */
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
/**
* Remove the specified file from the parity.
* The file is then inserted in the deleted set, and it should no be deallocated.
*/
static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
/* remove from the list of contained files */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* state changed */
state->need_write = 1;
/* free all the blocks of the file */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
block_off_t block_pos = block->parity_pos;
struct snapraid_block* block = fs_file2block_get(file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
unsigned block_state;
struct snapraid_deleted* deleted;
/* adjust the first free position */
/* note that doing all the deletions before alllocations, */
/* first_free_block is always 0 and the "if" is never triggered */
/* but we keep this code anyway for completeness. */
if (disk->first_free_block > block_pos)
disk->first_free_block = block_pos;
/* allocated a new deleted block from the block we are going to delete */
deleted = deleted_dup(block);
if (disk->first_free_block > parity_pos)
disk->first_free_block = parity_pos;
/* in case we scan after an aborted sync, */
/* we could get also intermediate states */
@ -205,93 +299,32 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
/* - File is now deleted after the aborted sync */
/* - Sync again, deleting the blocks (exactly here) */
/* with the hash of CHG block not represeting the real parity state */
hash_invalid_set(deleted->block.hash);
hash_invalid_set(block->hash);
}
break;
case BLOCK_STATE_REP :
/* we just don't know the old hash, and then we set it to invalid */
hash_invalid_set(deleted->block.hash);
hash_invalid_set(block->hash);
break;
default :
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in deallocating for block %u state %u\n", block->parity_pos, block_state);
log_fatal("Internal inconsistency in deallocating for block %u state %u\n", parity_pos, block_state);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* insert it in the list of deleted blocks */
tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted);
/* set the deleted block in the block array */
tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block);
}
}
/**
* Checks if a file is completely formed of blocks with invalid parity,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has an invalid parity */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* exclude blocks with parity */
if (!block_has_invalid_parity(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* if rehash fails */
if (info_get_rehash(info))
return 0;
/* set the block as deleted */
block_state_set(block, BLOCK_STATE_DELETED);
}
return 1;
/* mark the file as deleted */
file_flag_set(file, FILE_IS_DELETED);
/* insert it in the list of deleted blocks */
tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has a hash */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
struct snapraid_block* block = &file->blockvec[i];
snapraid_info info;
/* exclude blocks without hash */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, block->parity_pos);
/* exclude blocks needing a rehash */
if (info_get_rehash(info))
return 0;
}
return 1;
}
static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file)
static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
@ -312,9 +345,76 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid
}
}
/* insert in the delayed list */
tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file);
}
/**
* Checks if a file is completely formed of blocks with invalid parity,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has an invalid parity */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
/* exclude blocks with parity */
if (!block_has_invalid_parity(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* if rehash fails */
if (info_get_rehash(info))
return 0;
}
return 1;
}
/**
* Checks if a file is completely formed of blocks with an updated hash,
* and no rehash is tagged, and if it has at least one block.
*/
static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
/* with no block, it never has a hash */
if (file->blockmax == 0)
return 0;
/* check all blocks */
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = fs_file2block_get(file, i);
block_off_t parity_pos = fs_file2par_get(disk, file, i);
/* exclude blocks without hash */
if (!block_has_updated_hash(block))
return 0;
/* get block specific info */
info = info_get(&state->infoarr, parity_pos);
/* exclude blocks needing a rehash */
if (info_get_rehash(info))
return 0;
}
return 1;
}
/**
* Refresh the file info.
*
@ -399,7 +499,43 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
}
/**
* Keeps the file as it's (or with only a name/inode modification).
* Insert the file in the data set.
*/
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* insert the file in the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* delayed allocation of the parity */
scan_file_delayed_allocate(scan, file);
}
/**
* Remove the file from the data set.
*
* File is then deleted.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* remove the file from the containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE))
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
}
/**
* Keep the file as it's (or with only a name/inode modification).
*
* If the file is kept, nothing has to be done.
*
@ -414,139 +550,17 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* fil
struct snapraid_disk* disk = scan->disk;
/* if the file is full invalid, schedule a reinsert at later stage */
if (file_is_full_invalid_parity_and_stable(scan->state, file)) {
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) {
struct snapraid_file* copy = file_dup(file);
/* remove the file from the list */
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* remove the file */
scan_file_remove(scan, file);
/* insert the file in the delayed block allocation */
scan_file_delayed_insert(scan, file);
/* reinsert the copy in the delayed list */
scan_file_insert(scan, copy);
}
}
/**
* Removes the specified file from the data set.
*/
static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_disk* disk = scan->disk;
/* deallocate the file from the parity */
scan_file_deallocate(scan, file);
/* remove the file from the file containers */
if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) {
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
}
tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset);
tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset);
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* deallocate */
file_free(file);
}
/**
* Inserts the specified file in the data set.
*/
static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file)
{
struct snapraid_state* state = scan->state;
struct snapraid_disk* disk = scan->disk;
block_off_t i;
block_off_t block_max;
block_off_t block_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
block_pos = disk->first_free_block;
block_max = tommy_arrayblk_size(&disk->blockarr);
for (i = 0; i < file->blockmax; ++i) {
snapraid_info info;
struct snapraid_block* block = &file->blockvec[i];
/* increment the position until the first really free block */
while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
++block_pos;
/* if not found, allocate a new one */
if (block_pos == block_max) {
++block_max;
tommy_arrayblk_grow(&disk->blockarr, block_max);
}
/* set the position */
block->parity_pos = block_pos;
/* get block specific info */
info = info_get(&state->infoarr, block_pos);
/* if the file block already has an updated hash without rehash */
if (block_has_updated_hash(block) && !info_get_rehash(info)) {
/* the only possible case is for REP blocks */
assert(block_state_get(block) == BLOCK_STATE_REP);
/* convert to a REP block */
block_state_set(&file->blockvec[i], BLOCK_STATE_REP);
/* and keep the hash as it's */
} else {
struct snapraid_block* over_block;
unsigned block_state;
/* convert to a CHG block */
block_state_set(block, BLOCK_STATE_CHG);
/* block to overwrite */
over_block = tommy_arrayblk_get(&disk->blockarr, block_pos);
/* state of the block we are going to overwrite */
block_state = block_state_get(over_block);
/* if the block is an empty one */
if (block_state == BLOCK_STATE_EMPTY) {
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(block->hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* copy the past hash of the block */
memcpy(block->hash, over_block->hash, HASH_SIZE);
/* if we have not already cleared the past hash */
if (!state->clear_past_hash) {
/* in this case we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
}
}
/* store in the disk map, after invalidating all the other blocks */
tommy_arrayblk_set(&disk->blockarr, block_pos, block);
/* set the new free position */
disk->first_free_block = block_pos + 1;
}
/* note that the file is already added in the file hashtables */
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
/**
* Processes a file.
*/
@ -869,7 +883,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
/* if found, and it's a fully hashed file */
if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) {
/* assume that the file is a copy, and reuse the hash */
file_copy(other_file, file);
@ -909,13 +923,8 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
}
}
/* insert the file in the file hashtables, to allow to find duplicate hardlinks */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
/* insert the file in the delayed block allocation */
scan_file_delayed_insert(scan, file);
/* insert the file in the delayed list */
scan_file_insert(scan, file);
}
/**
@ -1544,8 +1553,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
/* next node */
node = node->next;
/* insert it */
scan_file_insert(scan, file);
/* insert in the parity */
scan_file_allocate(scan, file);
}
/* mark the disk without reliable physical offset if it has duplicates */
@ -1766,6 +1775,9 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
/* check the filesystem on all disks */
state_fscheck(state, "after scan");
if (is_diff) {
/* check for file difference */
if (!no_difference)

39
cmdline/scrub.c
View file

@ -184,6 +184,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
struct snapraid_block* block;
int file_is_unsynced;
struct snapraid_disk* disk = handle[j].disk;
struct snapraid_file* file;
block_off_t file_pos;
/* if the file on this disk is synced */
/* if not, silent errors are assumed as expected error */
@ -200,13 +202,16 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
}
/* if the block is not used */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
if (!block_has_file(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* get the file of this block */
file = fs_par2file_get(disk, i, &file_pos);
/* if the block is unsynced, errors are expected */
if (block_has_invalid_parity(block)) {
/* report that the block and the file are not synced */
@ -219,16 +224,16 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
state_usage_cpu(state);
/* if the file is different than the current one, close it */
if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
if (handle[j].file != 0 && handle[j].file != file) {
/* keep a pointer at the file we are going to close for error reporting */
struct snapraid_file* file = handle[j].file;
struct snapraid_file* report = handle[j].file;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
if (errno == EIO) {
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -236,7 +241,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
goto bail;
}
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to scrub.\n");
log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -246,10 +251,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
}
}
ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0);
ret = handle_open(&handle[j], file, state->file_mode, log_error, 0);
if (ret == -1) {
/* file we have tried to open for error reporting */
struct snapraid_file* file = block_file_get(block);
if (errno == EIO) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
@ -268,9 +271,9 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
}
/* check if the file is changed */
if (handle[j].st.st_size != block_file_get(block)->size
|| handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
if (handle[j].st.st_size != file->size
|| handle[j].st.st_mtime != file->mtime_sec
|| STAT_NSEC(&handle[j].st) != file->mtime_nsec
/* don't check the inode to support filesystem without persistent inodes */
) {
/* report that the block and the file are not synced */
@ -283,12 +286,10 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
/* from the last sync, as we are expected to return errors if running */
/* in an unsynced array. This is just like the check command. */
read_size = handle_read(&handle[j], block, buffer[j], state->block_size, log_error, 0);
read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, log_error, 0);
if (read_size == -1) {
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
if (errno == EIO) {
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
if (io_error >= state->opt.io_error_limit) {
/* LCOV_EXCL_START */
log_fatal("DANGER! Too many input/output read error in a data disk, it isn't possible to scrub.\n");
@ -299,13 +300,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
/* LCOV_EXCL_STOP */
}
log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, file_pos);
++io_error;
io_error_on_this_block = 1;
continue;
}
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
++error;
error_on_this_block = 1;
continue;
@ -332,14 +333,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
unsigned diff = memdiff(hash, block->hash, HASH_SIZE);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(handle[j].file->sub), block_file_pos(block), diff);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff);
/* it's a silent error only if we are dealing with synced files */
if (file_is_unsynced) {
++error;
error_on_this_block = 1;
} else {
log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, block_file_pos(block), diff);
log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, file_pos, diff);
++silent_error;
silent_error_on_this_block = 1;
}

8
cmdline/search.c
View file

@ -130,7 +130,7 @@ int search_file_compare(const void* void_arg, const void* void_data)
return 0;
}
int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_block* missing_block, unsigned char* buffer)
int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer)
{
struct snapraid_search_file* file;
tommy_uint32_t file_hash;
@ -138,10 +138,10 @@ int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapra
arg.state = state;
arg.block = missing_block;
arg.file = block_file_get(missing_block);
arg.file = missing_file;
arg.buffer = buffer;
arg.offset = state->block_size * (data_off_t)block_file_pos(missing_block);
arg.read_size = block_file_size(missing_block, state->block_size);
arg.offset = state->block_size * (data_off_t)missing_file_pos;
arg.read_size = file_block_size(missing_file, missing_file_pos, state->block_size);
arg.prevhash = prevhash;
file_hash = file_stamp_hash(arg.file->size, arg.file->mtime_sec, arg.file->mtime_nsec);

2
cmdline/search.h
View file

@ -48,7 +48,7 @@ void search_file_free(struct snapraid_search_file* file);
* Fetch a file from the size, timestamp and name.
* Return ==0 if the block is found, and copied into buffer.
*/
int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_block* missing_block, unsigned char* buffer);
int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer);
/**
* Import files from the specified directory.

41
cmdline/snapraid.c
View file

@ -78,14 +78,14 @@ void memory(void)
{
log_tag("memory:used:%" PRIu64 "\n", (uint64_t)malloc_counter());
/* size of the block + the size of the blockarr pointer to it */
log_tag("memory:block:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_block) + sizeof(void*)));
/* size of the file + the size of the 3 hashtables pointers to it, * 2 for the hashtable grow factor */
log_tag("memory:file:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_file) + 3*2*sizeof(void*)));
/* size of the block */
log_tag("memory:block:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_block)));
log_tag("memory:chunk:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_chunk)));
log_tag("memory:file:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_file)));
log_tag("memory:link:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_link)));
log_tag("memory:dir:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_dir)));
msg_progress("Using %u MiB of memory.\n", (unsigned)(malloc_counter() / (1024 * 1024)));
}
/****************************************************************************/
@ -234,7 +234,6 @@ void config(char* conf, size_t conf_size, const char* argv0)
#define OPT_TEST_FORCE_SCRUB_AT 271
#define OPT_TEST_FORCE_SCRUB_EVEN 272
#define OPT_TEST_FORCE_CONTENT_WRITE 273
#define OPT_TEST_FORCE_CONTENT_TEXT 274
#define OPT_TEST_SKIP_CONTENT_CHECK 275
#define OPT_TEST_SKIP_PARITY_ACCESS 276
#define OPT_TEST_EXPECT_FAILURE 277
@ -327,9 +326,6 @@ struct option long_options[] = {
/* Force write of the content file even if no modification is done */
{ "test-force-content-write", 0, 0, OPT_TEST_FORCE_CONTENT_WRITE },
/* Force the use of text content file */
{ "test-force-content-text", 0, 0, OPT_TEST_FORCE_CONTENT_TEXT },
/* Relax the checks done at the content file */
{ "test-skip-content-check", 0, 0, OPT_TEST_SKIP_CONTENT_CHECK },
@ -684,9 +680,6 @@ int main(int argc, char* argv[])
case OPT_TEST_FORCE_CONTENT_WRITE :
opt.force_content_write = 1;
break;
case OPT_TEST_FORCE_CONTENT_TEXT :
opt.force_content_text = 1;
break;
case OPT_TEST_EXPECT_FAILURE :
/* invert the exit codes */
exit_success = 1;
@ -1001,9 +994,29 @@ int main(int argc, char* argv[])
/* in the next state read ensures to clear all the past hashes in case */
/* we are reading from an incomplete sync */
/* The undeterminated hash are only for CHG/DELETED blocks for which we don't */
/* know if the previous interrupted sync was able to update or not the parity */
/* know if the previous interrupted sync was able to update or not the parity. */
/* The sync process instead needs to trust this information because it's used */
/* to avoid to recompute the parity if all the input are equals as before. */
/* In these cases we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the data that may be now */
/* deleted. Then we reset the hash to a bogus value. */
/* An example for CHG blocks is: */
/* - One file is added creating a CHG block with ZERO state */
/* - Sync aborted after updating the parity to the new state, */
/* but without saving the content file representing this new BLK state. */
/* - File is now deleted after the aborted sync */
/* - Sync again, deleting the blocks overt the CHG ones */
/* with the hash of CHG blocks not represeting the real parity state */
/* An example for DELETED blocks is: */
/* - One file is deleted creating DELETED blocks */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new EMPTY state. */
/* - Another file is added again over the DELETE ones */
/* with the hash of DELETED blocks not represeting the real parity state */
state.clear_past_hash = 1;
state_read(&state);

1653
cmdline/state.c
View file

File diff suppressed because it is too large Load diff

7
cmdline/state.h
View file

@ -98,7 +98,6 @@ struct snapraid_option {
unsigned force_scrub_at; /**< Force scrub for the specified number of blocks. */
int force_scrub_even; /**< Force scrub of all the even blocks. */
int force_content_write; /**< Force the update of the content file. */
int force_content_text; /**< Force the use of text version of content file. */
int force_scan_winfind; /**< Force the use of FindFirst/Next in Windows to list directories. */
int force_progress; /**< Force the use of the progress status. */
unsigned force_autosave_at; /**< Force autosave at the specified block. */
@ -373,6 +372,12 @@ static inline void state_usage_parity(struct snapraid_state* state, unsigned lev
*/
void state_usage_print(struct snapraid_state* state);
/**
* Check the filesystem on all disks.
* On error it aborts.
*/
void state_fscheck(struct snapraid_state* state, const char* ope);
/****************************************************************************/
/* misc */

12
cmdline/status.c
View file

@ -136,12 +136,13 @@ int state_status(struct snapraid_state* state)
/* check fragmentation */
if (file->blockmax != 0) {
block_off_t prev_pos;
block_off_t last_pos;
int fragmented;
fragmented = 0;
prev_pos = file->blockvec[0].parity_pos;
prev_pos = fs_file2par_get(disk, file, 0);
for (j = 1; j < file->blockmax; ++j) {
block_off_t parity_pos = file->blockvec[j].parity_pos;
block_off_t parity_pos = fs_file2par_get(disk, file, j);
if (prev_pos + 1 != parity_pos) {
fragmented = 1;
++extra_fragment;
@ -151,8 +152,9 @@ int state_status(struct snapraid_state* state)
}
/* keep track of latest block used */
if (file->blockvec[file->blockmax - 1].parity_pos > disk_block_latest_used) {
disk_block_latest_used = file->blockvec[file->blockmax - 1].parity_pos;
last_pos = fs_file2par_get(disk, file, file->blockmax - 1);
if (last_pos > disk_block_latest_used) {
disk_block_latest_used = last_pos;
}
if (fragmented) {
@ -279,7 +281,7 @@ int state_status(struct snapraid_state* state)
one_valid = 0;
for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) {
struct snapraid_disk* disk = node_disk->data;
struct snapraid_block* block = disk_block_get(disk, i);
struct snapraid_block* block = fs_par2block_get(disk, i);
if (block_has_file(block))
one_valid = 1;

228
cmdline/stream.c
View file

@ -21,31 +21,6 @@
#include "util.h"
#include "stream.h"
/****************************************************************************/
/* hex conversion table */
static char strhexset[16] = "0123456789abcdef";
static unsigned strdecset[256] =
{
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1
};
/****************************************************************************/
/* stream */
@ -470,108 +445,6 @@ int sgetu32(STREAM* f, uint32_t* value)
}
}
int sgetu64(STREAM* f, uint64_t* value)
{
int c;
c = sgetc(f);
if (c >= '0' && c <= '9') {
uint64_t v;
v = c - '0';
c = sgetc(f);
while (c >= '0' && c <= '9') {
v *= 10;
v += c - '0';
c = sgetc(f);
}
*value = v;
sungetc(c, f);
return 0;
} else {
/* LCOV_EXCL_START */
/* nothing read */
return -1;
/* LCOV_EXCL_STOP */
}
}
int sgethex(STREAM* f, void* void_data, int size)
{
unsigned char* data = void_data;
/* if there is enough data in memory */
if (sptrlookup(f, size * 2)) {
/* optimized version with all the data in memory */
unsigned char* pos = sptrget(f);
unsigned x = 0;
while (size--) {
unsigned b0;
unsigned b1;
unsigned b;
b0 = strdecset[pos[0]];
b1 = strdecset[pos[1]];
pos += 2;
b = (b0 << 4) | b1;
x |= b;
*data++ = b;
}
/* at the end check if a digit was wrong */
if (x > 0xFF) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
sptrset(f, pos);
} else {
/* standard version using sgetc() */
while (size--) {
unsigned b0;
unsigned b1;
unsigned b;
int c;
c = sgetc(f);
if (c == EOF) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
b0 = strdecset[(unsigned char)c];
c = sgetc(f);
if (c == EOF) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
b1 = strdecset[(unsigned char)c];
b = (b0 << 4) | b1;
if (b > 0xFF) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
*data++ = b;
}
}
return 0;
}
int sgetb32(STREAM* f, uint32_t* value)
{
uint32_t v;
@ -680,19 +553,6 @@ int sgetbs(STREAM* f, char* str, int size)
return sread(f, str, (int)len);
}
int sputs(const char* str, STREAM* f)
{
while (*str) {
if (sputc(*str++, f) != 0) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
}
return 0;
}
int swrite(const void* void_data, unsigned size, STREAM* f)
{
const unsigned char* data = void_data;
@ -723,94 +583,6 @@ int swrite(const void* void_data, unsigned size, STREAM* f)
return 0;
}
int sputu32(uint32_t value, STREAM* s)
{
char buf[16];
int i;
if (!value)
return sputc('0', s);
i = sizeof(buf);
while (value) {
buf[--i] = (value % 10) + '0';
value /= 10;
}
return swrite(buf + i, sizeof(buf) - i, s);
}
int sputu64(uint64_t value, STREAM* s)
{
char buf[32];
uint32_t value32;
int i;
if (!value)
return sputc('0', s);
i = sizeof(buf);
while (value > 0xFFFFFFFF) {
buf[--i] = (value % 10) + '0';
value /= 10;
}
value32 = (uint32_t)value;
while (value32) {
buf[--i] = (value32 % 10) + '0';
value32 /= 10;
}
return swrite(buf + i, sizeof(buf) - i, s);
}
int sputhex(const void* void_data, int size, STREAM* f)
{
const unsigned char* data = void_data;
/* if there is enough space in memory */
if (sptrlookup(f, size * 2)) {
/* optimized version with all the data in memory */
unsigned char* pos = sptrget(f);
while (size) {
unsigned b = *data;
*pos++ = strhexset[b >> 4];
*pos++ = strhexset[b & 0xF];
++data;
--size;
}
sptrset(f, pos);
} else {
/* standard version using sputc() */
while (size) {
unsigned b = *data;
if (sputc(strhexset[b >> 4], f) != 0) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
if (sputc(strhexset[b & 0xF], f) != 0) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
++data;
--size;
}
}
return 0;
}
int sputb32(uint32_t value, STREAM* s)
{
unsigned char b;

43
cmdline/stream.h
View file

@ -305,19 +305,6 @@ int sgetlasttok(STREAM* f, char* str, int size);
*/
int sgetu32(STREAM* f, uint32_t* value);
/**
* Reads a 64 bit number.
* Stops at the first not digit char.
* Returns <0 if there isn't enough to read.
*/
int sgetu64(STREAM* f, uint64_t* value);
/**
* Reads an hexadecimal string of fixed length.
* Returns <0 if there isn't enough to read.
*/
int sgethex(STREAM* f, void* data, int size);
/****************************************************************************/
/* binary get */
@ -376,42 +363,12 @@ static inline int sputeol(STREAM* s)
return sputc('\n', s);
}
/**
* Writes a string.
* Returns 0 on success or -1 on error.
*/
int sputs(const char* str, STREAM* f);
/**
* Write a sized string.
* Returns 0 on success or -1 on error.
*/
int swrite(const void* data, unsigned size, STREAM* f);
/**
* Write a string literal (constant string).
* Returns 0 on success or -1 on error.
*/
#define sputsl(str, f) swrite(str, sizeof(str) - 1, f)
/**
* Writes a 32 bit number.
* Returns 0 on success or -1 on error.
*/
int sputu32(uint32_t value, STREAM* s);
/**
* Writes a 64 bit number.
* Returns 0 on success or -1 on error.
*/
int sputu64(uint64_t value, STREAM* s);
/**
* Writes a hexadecimal string of fixed length.
* Returns 0 on success or -1 on error.
*/
int sputhex(const void* void_data, int size, STREAM* s);
/****************************************************************************/
/* binary put */

11
cmdline/support.c
View file

@ -597,6 +597,9 @@ int fmtime(int f, int64_t mtime_sec, int mtime_nsec)
/****************************************************************************/
/* memory */
/**
* Total amount of memory allocated.
*/
static size_t mcounter;
size_t malloc_counter(void)
@ -604,6 +607,7 @@ size_t malloc_counter(void)
return mcounter;
}
/* LCOV_EXCL_START */
static ssize_t malloc_print(int f, const char* str)
{
ssize_t len = 0;
@ -611,7 +615,9 @@ static ssize_t malloc_print(int f, const char* str)
++len;
return write(f, str, len);
}
/* LCOV_EXCL_STOP */
/* LCOV_EXCL_START */
static ssize_t malloc_printn(int f, size_t value)
{
char buf[32];
@ -628,11 +634,12 @@ static ssize_t malloc_printn(int f, size_t value)
return write(f, buf + i, sizeof(buf) - i);
}
/* LCOV_EXCL_STOP */
/* LCOV_EXCL_START */
void malloc_fail(size_t size)
{
/* don't use printf to avoid any possible extra allocation */
/* LCOV_EXCL_START */
int f = 2; /* stderr */
malloc_print(f, "Failed for Low Memory!\n");
@ -646,8 +653,8 @@ void malloc_fail(size_t size)
malloc_print(f, "You are currently using a 32 bits executable.\n");
malloc_print(f, "If you have more than 4GB of memory, please upgrade to a 64 bits one.\n");
}
/* LCOV_EXCL_STOP */
}
/* LCOV_EXCL_STOP */
void* malloc_nofail(size_t size)
{

111
cmdline/sync.c
View file

@ -65,7 +65,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
struct snapraid_block* block;
unsigned block_state;
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -100,8 +100,10 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
int read_size;
unsigned char hash[HASH_SIZE];
unsigned block_state;
struct snapraid_file* file;
block_off_t file_pos;
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -110,6 +112,9 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
if (block_state != BLOCK_STATE_REP && block_state != BLOCK_STATE_CHG)
continue;
/* get the file of this block */
file = fs_par2file_get(disk, i, &file_pos);
/* get block specific info */
info = info_get(&state->infoarr, i);
@ -120,16 +125,16 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
state_usage_cpu(state);
/* if the file is different than the current one, close it */
if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
if (handle[j].file != 0 && handle[j].file != file) {
/* keep a pointer at the file we are going to close for error reporting */
struct snapraid_file* file = handle[j].file;
struct snapraid_file* report = handle[j].file;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
if (errno == EIO) {
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -137,7 +142,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
goto bail;
}
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -147,10 +152,8 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
}
}
ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0);
ret = handle_open(&handle[j], file, state->file_mode, log_error, 0);
if (ret == -1) {
/* file we have tried to open for error reporting */
struct snapraid_file* file = block_file_get(block);
if (errno == EIO) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
@ -194,13 +197,11 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
}
/* check if the file is changed */
if (handle[j].st.st_size != block_file_get(block)->size
|| handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
|| handle[j].st.st_ino != block_file_get(block)->inode
if (handle[j].st.st_size != file->size
|| handle[j].st.st_mtime != file->mtime_sec
|| STAT_NSEC(&handle[j].st) != file->mtime_nsec
|| handle[j].st.st_ino != file->inode
) {
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc(file->sub));
if (handle[j].st.st_size != file->size) {
log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, handle[j].st.st_size);
@ -218,13 +219,11 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
continue;
}
read_size = handle_read(&handle[j], block, buffer, state->block_size, log_fatal, 0);
read_size = handle_read(&handle[j], file_pos, buffer, state->block_size, log_fatal, 0);
if (read_size == -1) {
/* LCOV_EXCL_START */
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
if (errno == EIO) {
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -232,7 +231,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
goto bail;
}
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that file '%s' can be read.\n", handle[j].path);
log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub);
@ -256,11 +255,8 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
if (block_state == BLOCK_STATE_REP) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
log_tag("error:%u:%s:%s: Unexpected data change\n", i, disk->name, esc(file->sub));
log_error("Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
log_error("Data change at file '%s' at position '%u'\n", handle[j].path, file_pos);
log_error("WARNING! Unexpected data modification of a file without parity!\n");
if (file_flag_has(file, FILE_IS_COPY)) {
@ -308,14 +304,14 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
/* close the last file in the disk */
if (handle[j].file != 0) {
/* keep a pointer at the file we are going to close for error reporting */
struct snapraid_file* file = handle[j].file;
struct snapraid_file* report = handle[j].file;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
if (errno == EIO) {
log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockmax, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockmax, disk->name, esc(report->sub), strerror(errno));
log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", blockmax);
@ -323,7 +319,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
goto bail;
}
log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc(report->sub), strerror(errno));
log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
log_fatal("Stopping at block %u\n", blockmax);
@ -424,7 +420,7 @@ static int block_is_enabled(block_off_t i, struct snapraid_handle* handle, unsig
if (!handle[j].disk)
continue;
block = disk_block_get(handle[j].disk, i);
block = fs_par2block_get(handle[j].disk, i);
if (block_has_file(block))
one_valid = 1;
@ -572,6 +568,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
struct snapraid_block* block;
unsigned block_state;
struct snapraid_disk* disk = handle[j].disk;
struct snapraid_file* file;
block_off_t file_pos;
/* by default no rehash in case of "continue" */
rehandle[j].block = 0;
@ -584,7 +582,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
/* get the block */
block = disk_block_get(disk, i);
block = fs_par2block_get(disk, i);
/* get the state of the block */
block_state = block_state_get(block);
@ -621,20 +619,23 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
continue;
}
/* get the file of this block */
file = fs_par2file_get(disk, i, &file_pos);
/* until now is CPU */
state_usage_cpu(state);
/* if the file is different than the current one, close it */
if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
if (handle[j].file != 0 && handle[j].file != file) {
/* keep a pointer at the file we are going to close for error reporting */
struct snapraid_file* file = handle[j].file;
struct snapraid_file* report = handle[j].file;
ret = handle_close(&handle[j]);
if (ret == -1) {
/* LCOV_EXCL_START */
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
if (errno == EIO) {
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -642,7 +643,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
goto bail;
}
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno));
log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path);
log_fatal("Stopping at block %u\n", i);
@ -652,10 +653,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
}
ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0);
ret = handle_open(&handle[j], file, state->file_mode, log_error, 0);
if (ret == -1) {
/* file we have tried to open for error reporting */
struct snapraid_file* file = block_file_get(block);
if (errno == EIO) {
/* LCOV_EXCL_START */
log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno));
@ -701,13 +700,11 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
/* check if the file is changed */
if (handle[j].st.st_size != block_file_get(block)->size
|| handle[j].st.st_mtime != block_file_get(block)->mtime_sec
|| STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec
|| handle[j].st.st_ino != block_file_get(block)->inode
if (handle[j].st.st_size != file->size
|| handle[j].st.st_mtime != file->mtime_sec
|| STAT_NSEC(&handle[j].st) != file->mtime_nsec
|| handle[j].st.st_ino != file->inode
) {
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc(file->sub));
if (handle[j].st.st_size != file->size) {
log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, handle[j].st.st_size);
@ -726,13 +723,11 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
continue;
}
read_size = handle_read(&handle[j], block, buffer[j], state->block_size, log_error, 0);
read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, log_error, 0);
if (read_size == -1) {
/* LCOV_EXCL_START */
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
if (errno == EIO) {
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
if (io_error >= state->opt.io_error_limit) {
log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path);
@ -741,13 +736,13 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
goto bail;
}
log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, file_pos);
++io_error;
io_error_on_this_block = 1;
continue;
}
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno));
log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno));
log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n");
log_fatal("Ensure that file '%s' can be read.\n", handle[j].path);
log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub);
@ -775,13 +770,10 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (block_has_updated_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* file we are processing for error reporting */
struct snapraid_file* file = block_file_get(block);
/* if the file has invalid parity, it's a REP changed during the sync */
if (block_has_invalid_parity(block)) {
log_tag("error:%u:%s:%s: Unexpected data change\n", i, disk->name, esc(file->sub));
log_error("Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
log_error("Data change at file '%s' at position '%u'\n", handle[j].path, file_pos);
log_error("WARNING! Unexpected data modification of a file without parity!\n");
if (file_flag_has(file, FILE_IS_COPY)) {
@ -801,8 +793,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
continue;
} else { /* otherwise it's a BLK with silent error */
unsigned diff = memdiff(hash, block->hash, HASH_SIZE);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), block_file_pos(block), diff);
log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, block_file_pos(block), diff);
log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff);
log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, file_pos, diff);
/* save the failed block for the fix */
failed[failed_count].index = j;
@ -1027,9 +1019,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (!io_error_on_this_block) {
/* for each disk, mark the blocks as processed */
for (j = 0; j < diskmax; ++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
struct snapraid_block* block;
if (!handle[j].disk)
continue;
block = fs_par2block_get(handle[j].disk, i);
if (block == BLOCK_EMPTY) {
/* nothing to do */
@ -1039,7 +1034,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* if it's a deleted block */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* the parity is now updated without this block, so it's now empty */
tommy_arrayblk_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
fs_deallocate(handle[j].disk, i);
continue;
}

11
snapraid.1
View file

@ -1268,15 +1268,14 @@ A reason to use a different blocksize is if your system has less
than 4 GiB of memory. As a rule of thumb, with 4 GiB or more memory
use the default 256, with 2 GiB use 512, and with 1 GiB use 1024.
.PP
In more details SnapRAID requires about TS*28/BS bytes
of RAM memory to run in the 32 bits version, and TS*36/BS
in the 64 bits one. Where TS is the total size in bytes of
your disk array, and BS is the block size in bytes.
In more details SnapRAID requires about TS*17/BS bytes of RAM memory.
Where TS is the total size in bytes of your disk array, and BS is the
block size in bytes.
.PP
For example with 8 disk of 4 TB and a block size of 256 KiB
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes) you have:
.PP
RAM = (8 * 4 * 10^12) * 28 / (256 * 2^10) = 3.2 GiB
RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB
.PD 0
.PP
.PD

11
snapraid.d
View file

@ -985,15 +985,14 @@ Configuration
than 4 GiB of memory. As a rule of thumb, with 4 GiB or more memory
use the default 256, with 2 GiB use 512, and with 1 GiB use 1024.
In more details SnapRAID requires about TS*28/BS bytes
of RAM memory to run in the 32 bits version, and TS*36/BS
in the 64 bits one. Where TS is the total size in bytes of
your disk array, and BS is the block size in bytes.
In more details SnapRAID requires about TS*17/BS bytes of RAM memory.
Where TS is the total size in bytes of your disk array, and BS is the
block size in bytes.
For example with 8 disk of 4 TB and a block size of 256 KiB
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes) you have:
:RAM = (8 * 4 * 10^12) * 28 / (256 * 2^10) = 3.2 GiB
:RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB
Another reason to use a different blocksize is if you have a lot of
small files. In the order of many millions.

11
snapraid.txt
View file

@ -1063,15 +1063,14 @@ A reason to use a different blocksize is if your system has less
than 4 GiB of memory. As a rule of thumb, with 4 GiB or more memory
use the default 256, with 2 GiB use 512, and with 1 GiB use 1024.
In more details SnapRAID requires about TS*28/BS bytes
of RAM memory to run in the 32 bits version, and TS*36/BS
in the 64 bits one. Where TS is the total size in bytes of
your disk array, and BS is the block size in bytes.
In more details SnapRAID requires about TS*17/BS bytes of RAM memory.
Where TS is the total size in bytes of your disk array, and BS is the
block size in bytes.
For example with 8 disk of 4 TB and a block size of 256 KiB
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes) you have:
RAM = (8 * 4 * 10^12) * 28 / (256 * 2^10) = 3.2 GiB
RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB
Another reason to use a different blocksize is if you have a lot of
small files. In the order of many millions.

1
tommyds/tommy.c
View file

@ -38,5 +38,6 @@
#include "tommyarrayblk.c"
#include "tommyarrayblkof.c"
#include "tommylist.c"
#include "tommytree.c"
#include "tommyhashdyn.c"

283
tommyds/tommytree.c Normal file
View file

@ -0,0 +1,283 @@
/*
* Copyright (c) 2015, Andrea Mazzoleni. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#include "tommytree.h"
#include <assert.h> /* for assert */
/******************************************************************************/
/* tree */
void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp)
{
tree->root = 0;
tree->count = 0;
tree->cmp = cmp;
}
static int tommy_tree_delta(tommy_tree_node* root)
{
int left_height = root->prev ? root->prev->key : 0;
int right_height = root->next ? root->next->key : 0;
return left_height - right_height;
}
/* AVL tree operations */
static tommy_tree_node* tommy_tree_balance(tommy_tree_node*);
static tommy_tree_node* tommy_tree_rotate_left(tommy_tree_node* root)
{
tommy_tree_node* next = root->next;
root->next = next->prev;
next->prev = tommy_tree_balance(root);
return tommy_tree_balance(next);
}
static tommy_tree_node* tommy_tree_rotate_right(tommy_tree_node* root)
{
tommy_tree_node* prev = root->prev;
root->prev = prev->next;
prev->next = tommy_tree_balance(root);
return tommy_tree_balance(prev);
}
static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_node* node)
{
if (!root)
return node;
root->next = tommy_tree_move_right(root->next, node);
return tommy_tree_balance(root);
}
static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root)
{
int delta = tommy_tree_delta(root);
if (delta < -1) {
if (tommy_tree_delta(root->next) > 0)
root->next = tommy_tree_rotate_right(root->next);
return tommy_tree_rotate_left(root);
}
if (delta > 1) {
if (tommy_tree_delta(root->prev) < 0)
root->prev = tommy_tree_rotate_left(root->prev);
return tommy_tree_rotate_right(root);
}
/* recompute key */
root->key = 0;
if (root->prev && root->prev->key > root->key)
root->key = root->prev->key;
if (root->next && root->next->key > root->key)
root->key = root->next->key;
/* count itself */
root->key += 1;
return root;
}
static tommy_tree_node* tommy_tree_insert_node(tommy_compare_func* cmp, tommy_tree_node* root, tommy_tree_node** let)
{
int c;
if (!root)
return *let;
c = cmp((*let)->data, root->data);
if (c < 0) {
root->prev = tommy_tree_insert_node(cmp, root->prev, let);
return tommy_tree_balance(root);
}
if (c > 0) {
root->next = tommy_tree_insert_node(cmp, root->next, let);
return tommy_tree_balance(root);
}
/* already present, set the return pointer */
*let = root;
return root;
}
void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data)
{
node->data = data;
node->prev = 0;
node->next = 0;
node->key = 0;
tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &node);
++tree->count;
return node->data;
}
static tommy_tree_node* tommy_tree_remove_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data, tommy_tree_node** let)
{
int c;
if (!root)
return 0;
c = cmp(data, root->data);
if (c < 0) {
root->prev = tommy_tree_remove_node(cmp, root->prev, data, let);
return tommy_tree_balance(root);
}
if (c > 0) {
root->next = tommy_tree_remove_node(cmp, root->next, data, let);
return tommy_tree_balance(root);
}
/* found */
*let = root;
return tommy_tree_move_right(root->prev, root->next);
}
void* tommy_tree_remove(tommy_tree* tree, void* data)
{
tommy_tree_node* node = 0;
tree->root = tommy_tree_remove_node(tree->cmp, tree->root, data, &node);
if (!node)
return 0;
return node->data;
}
static tommy_tree_node* tommy_tree_search_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data)
{
int c;
if (!root)
return 0;
c = cmp(data, root->data);
if (c < 0)
return tommy_tree_search_node(cmp, root->prev, data);
if (c > 0)
return tommy_tree_search_node(cmp, root->next, data);
return root;
}
void* tommy_tree_search(tommy_tree* tree, void* data)
{
tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data);
if (!node)
return 0;
return node->data;
}
void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg)
{
tommy_tree_node* node = tommy_tree_search_node(cmp, tree->root, cmp_arg);
if (!node)
return 0;
return node->data;
}
void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node)
{
return tommy_tree_remove(tree, node->data);
}
static void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func)
{
tommy_tree_node* next;
if (!root)
return;
tommy_tree_foreach_node(root->prev, func);
/* make a copy in case func is free() */
next = root->next;
func(root->data);
tommy_tree_foreach_node(next, func);
}
void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func)
{
tommy_tree_foreach_node(tree->root, func);
}
static void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg)
{
tommy_tree_node* next;
if (!root)
return;
tommy_tree_foreach_arg_node(root->prev, func, arg);
/* make a copy in case func is free() */
next = root->next;
func(arg, root->data);
tommy_tree_foreach_arg_node(next, func, arg);
}
void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg)
{
tommy_tree_foreach_arg_node(tree->root, func, arg);
}
tommy_size_t tommy_tree_memory_usage(tommy_tree* tree)
{
return tommy_tree_count(tree) * sizeof(tommy_tree_node);
}

214
tommyds/tommytree.h Normal file
View file

@ -0,0 +1,214 @@
/*
* Copyright (c) 2015, Andrea Mazzoleni. All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
/** \file
* AVL tree.
*
* This tree is a standard AVL tree implementation that stores elements in the order defined
* by the comparsion function.
*
* \code
* tommy_tree tree;
*
* tommy_tree_init(&tree, cmp);
* \endcode
*
* To insert elements in the tree you have to call tommy_tree_insert() for
* each element.
* In the insertion call you have to specify the address of the node, the
* address of the object.
* The address of the object is used to initialize the tommy_node::data field
* of the node.
*
* \code
* struct object {
* int value;
* // other fields
* tommy_tree_node node;
* };
*
* struct object* obj = malloc(sizeof(struct object)); // creates the object
*
* obj->value = ...; // initializes the object
*
* tommy_tree_insert(&tree, &obj->node, obj); // inserts the object
* \endcode
*
* To find and element in the tree you have to call tommy_tree_search() providing
* the key to search.
*
* \code
* struct object value_to_find = { 1 };
* struct object* obj = tommy_tree_search(&tree, &value_to_find);
* if (!obj) {
* // not found
* } else {
* // found
* }
* \endcode
*
* To remove an element from the tree you have to call tommy_tree_remove()
* providing the key to search and remove.
*
 * \code
* struct object* obj = tommy_tree_remove(&tree, value_to_remove);
* if (obj) {
* free(obj); // frees the object allocated memory
* }
* \endcode
*
* To destroy the tree you have to remove all the elements.
*
* If you need to iterate over all the elements in the tree, you can use
* tommy_hashtable_foreach() or tommy_hashtable_foreach_arg().
* If you need a more precise control with a real iteration, you have to insert
* all the elements also in a ::tommy_list, and use the list to iterate.
* See the \ref multiindex example for more detail.
*/
#ifndef __TOMMYTREE_H
#define __TOMMYTREE_H
#include "tommytypes.h"
/******************************************************************************/
/* tree */
/**
* Tree node.
* This is the node that you have to include inside your objects.
*/
typedef tommy_node tommy_tree_node;
typedef struct tommy_tree_struct {
tommy_tree_node* root; /**< Root node. */
tommy_count_t count; /**< Number of elements. */
tommy_compare_func* cmp; /**< Comparison function. */
} tommy_tree;
/**
* Initializes the tree.
*/
void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp);
/**
* Inserts an element in the tree.
* If the element is already present, it's not inserted again.
* You have to provide the pointer of the node embedded into the object,
* the pointer at the object.
* \param node Pointer at the node embedded into the object to insert.
* \param data Pointer at the object to insert.
* \return The element in the tree. Either the already existing one, or the one just inserted.
*/
void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data);
/**
* Searches and removes an element.
* If the element is not found, 0 is returned.
* \param data Element used for comparison.
* \return The removed element, or 0 if not found.
*/
void* tommy_tree_remove(tommy_tree* tree, void* data);
/**
* Searches an element in the tree.
* If the element is not found, 0 is returned.
* \param data Element used for comparison.
* \return The first element found, or 0 if none.
*/
void* tommy_tree_search(tommy_tree* tree, void* data);
/**
* Searches an element in the tree with a specific compare function.
*
* Like tommy_tree_search() but you can specify a different comparison function.
* Note that this function must define a suborder of the original one.
*
* The ::data argument is going to always be the first argument of the comparison function,
* and it can be something different than the objects in the tree.
*/
void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg);
/**
* Removes an element from the tree.
* You must already have the address of the element to remove.
* \return The tommy_node::data field of the node removed.
*/
void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node);
/**
* Calls the specified function for each element in the tree.
*
* The elements are processed in order.
*
* You can use this function to deallocate all the elements inserted.
*
* \code
* tommy_tree tree;
*
* // initializes the tree
* tommy_tree_init(&tree, cmp);
*
* ...
*
* // creates an object
* struct object* obj = malloc(sizeof(struct object));
*
* ...
*
* // insert it in the hashtable
* tommy_tree_insert(&tree, &obj->node, obj);
*
* ...
*
* // deallocates all the objects iterating the tree
* tommy_tree_foreach(&tree, free);
* \endcode
*/
void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func);
/**
* Calls the specified function with an argument for each element in the tree.
*/
void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg);
/**
* Gets the number of elements.
*/
tommy_inline tommy_count_t tommy_tree_count(tommy_tree* tree)
{
return tree->count;
}
/**
* Gets the size of allocated memory.
* It includes the size of the ::tommy_tree_node of the stored elements.
*/
tommy_size_t tommy_tree_memory_usage(tommy_tree* tree);
#endif