rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add an extensive parity filesystem check

Browse source 
With the new mapping it's now possible to check everything
in a very fast way.
This commit is contained in:
Andrea Mazzoleni 2015-10-05 21:40:44 +02:00
parent 74ca71600d
commit b17e618d49
6 changed files with 260 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

208
cmdline/elem.c
View file

@ -771,7 +771,7 @@ struct chunk_disk_empty {
};
/**
* Search for any block inside the specified blockmax.
* Compare the chunk if inside the specified blockmax.
*/
static int chunk_disk_empty_compare(const void* void_a, const void* void_b)
{
@ -812,7 +812,7 @@ struct chunk_disk_size {
};
/**
* Search for the chunk with the highest parity position.
* Compare the chunk by highest parity position.
*
* The maximum parity position is stored as size.
*/
@ -836,12 +836,179 @@ block_off_t disk_size(struct snapraid_disk* disk)
return arg.size;
}
struct chunk_check {
const struct snapraid_chunk* prev;
int result;
};
void chunk_parity_check_foreach(void* void_arg, void* void_obj)
{
struct chunk_check* arg = void_arg;
const struct snapraid_chunk* obj = void_obj;
const struct snapraid_chunk* prev = arg->prev;
/* set the next previous block */
arg->prev = obj;
/* stop reporting if too many errors */
if (arg->result > 100) {
/* LCOV_EXCL_START */
return;
/* LCOV_EXCL_STOP */
}
/* check only if there is a previous block */
if (!prev)
return;
/* check the order */
if (prev->parity_pos >= obj->parity_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in parity order for files '%s' at '%u:%u' and '%s' at '%u:%u'\n",
prev->file->sub, prev->parity_pos, prev->count, obj->file->sub, obj->parity_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
/* check that the chunks don't overlap */
if (prev->parity_pos + prev->count > obj->parity_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for parity overlap for files '%s' at '%u:%u' and '%s' at '%u:%u'\n",
prev->file->sub, prev->parity_pos, prev->count, obj->file->sub, obj->parity_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
void chunk_file_check_foreach(void* void_arg, void* void_obj)
{
struct chunk_check* arg = void_arg;
const struct snapraid_chunk* obj = void_obj;
const struct snapraid_chunk* prev = arg->prev;
/* set the next previous block */
arg->prev = obj;
/* stop reporting if too many errors */
if (arg->result > 100) {
/* LCOV_EXCL_START */
return;
/* LCOV_EXCL_STOP */
}
/* note that for deleted files, some chunks may be missing */
/* if the files are different */
if (!prev || prev->file != obj->file) {
if (prev != 0) {
if (file_flag_has(prev->file, FILE_IS_DELETED)) {
/* check that the chunk doesn't overflow the file */
if (prev->file_pos + prev->count > prev->file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in delete end for file '%s' at '%u:%u' overflowing size '%u'\n",
prev->file->sub, prev->file_pos, prev->count, prev->file->blockmax);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
} else {
/* check that the chunk ends the file */
if (prev->file_pos + prev->count != prev->file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file end for file '%s' at '%u:%u' instead of size '%u'\n",
prev->file->sub, prev->file_pos, prev->count, prev->file->blockmax);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
}
if (file_flag_has(obj->file, FILE_IS_DELETED)) {
/* check that the chunk doesn't overflow the file */
if (obj->file_pos + obj->count > obj->file->blockmax) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in delete start for file '%s' at '%u:%u' overflowing size '%u'\n",
obj->file->sub, obj->file_pos, obj->count, obj->file->blockmax);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
} else {
/* check that the chunk starts the file */
if (obj->file_pos != 0) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file start for file '%s' at '%u:%u'\n",
obj->file->sub, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
} else {
/* check the order */
if (prev->file_pos >= obj->file_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file order for file '%s' at '%u:%u' and at '%u:%u'\n",
prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
if (file_flag_has(obj->file, FILE_IS_DELETED)) {
/* check that the chunks don't overlap */
if (prev->file_pos + prev->count > obj->file_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in delete sequence for file '%s' at '%u:%u' and at '%u:%u'\n",
prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
} else {
/* check that the chunks are sequential */
if (prev->file_pos + prev->count != obj->file_pos) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in file sequence for file '%s' at '%u:%u' and at '%u:%u'\n",
prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count);
++arg->result;
return;
/* LCOV_EXCL_STOP */
}
}
}
}
int fs_check(struct snapraid_disk* disk)
{
struct chunk_check arg;
/* error count starts from 0 */
arg.result = 0;
/* check parity sequence */
arg.prev = 0;
tommy_tree_foreach_arg(&disk->fs_parity, chunk_parity_check_foreach, &arg);
/* check file sequence */
arg.prev = 0;
tommy_tree_foreach_arg(&disk->fs_file, chunk_file_check_foreach, &arg);
if (arg.result != 0)
return -1;
return 0;
}
struct chunk_parity_inside {
block_off_t parity_pos;
};
/**
* Search for the chunk containing the specified parity position.
* Compare the chunk if containing the specified parity position.
*/
static int chunk_parity_inside_compare(const void* void_a, const void* void_b)
{
@ -856,6 +1023,11 @@ static int chunk_parity_inside_compare(const void* void_a, const void* void_b)
return 0;
}
/**
* Seach the chunk at the specified parity position.
* The search is optimized for sequential accesses.
* \return If not found return 0
*/
static struct snapraid_chunk* fs_par2chunk_get(struct snapraid_disk* disk, block_off_t parity_pos)
{
struct snapraid_chunk* chunk;
@ -886,7 +1058,7 @@ struct chunk_file_inside {
};
/**
* Search for the chunk containing the specified file position.
* Compare the chunk if containing the specified file position.
*/
static int chunk_file_inside_compare(const void* void_a, const void* void_b)
{
@ -906,6 +1078,11 @@ static int chunk_file_inside_compare(const void* void_a, const void* void_b)
return 0;
}
/**
* Seach the chunk at the specified file position.
* The search is optimized for sequential accesses.
* \return If not found return 0
*/
static struct snapraid_chunk* fs_file2chunk_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos)
{
struct snapraid_chunk* chunk;
@ -952,7 +1129,7 @@ block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* fi
chunk = fs_file2chunk_get(disk, file, file_pos);
if (!chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for a file without parity in disk '%s'\n", disk->name);
log_fatal("Internal inconsistency when resolving a file to a not existing chunk in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@ -963,16 +1140,18 @@ block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* fi
void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos)
{
struct snapraid_chunk* chunk;
struct snapraid_chunk* parity_chunk;
struct snapraid_chunk* file_chunk;
if (file_pos > 0) {
/* search an existing chunk for the previous file_pos */
chunk = fs_file2chunk_get(disk, file, file_pos - 1);
if (chunk != 0 && parity_pos == chunk->parity_pos + chunk->count) {
/* ensure that we are really extending a chunk */
/* ensure that we are extending the chunk at the end */
if (file_pos != chunk->file_pos + chunk->count) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency extending a chunk in disk '%s'\n", disk->name);
log_fatal("Internal inconsistency extending a chunk in the middle in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@ -988,8 +1167,15 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap
chunk = chunk_alloc(parity_pos, file, file_pos, 1);
/* insert te chunk in the trees */
tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk);
tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk);
parity_chunk = tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk);
file_chunk = tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk);
if (parity_chunk != chunk || file_chunk != chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency when allocating a chunk that already exists in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* store the last accessed chunk */
disk->fs_last = chunk;
@ -1002,7 +1188,7 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos)
chunk = fs_par2chunk_get(disk, parity_pos);
if (!chunk) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for clearing a not existing block in disk '%s'\n", disk->name);
log_fatal("Internal inconsistency when deallocating a not existing chunk in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@ -1036,7 +1222,7 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos)
}
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for clearing in the middle of a chunk in disk '%s'\n", disk->name);
log_fatal("Internal inconsistency when deallocating a chunk in the middle in disk '%s'\n", disk->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}

16
cmdline/elem.h
View file

@ -229,6 +229,16 @@ struct snapraid_block {
*/
#define FILE_IS_WITHOUT_INODE 0x200
/**
* The file is deleted.
* This happens when a file is deleted from the array,
* but it's keep inside the parity until the next sync.
*
* During the filesystem check we needs this information,
* because deleted files may be present only partially.
*/
#define FILE_IS_DELETED 0x400
#define FILE_IS_HARDLINK 0x1000 /**< If it's an hardlink. */
#define FILE_IS_SYMLINK 0x2000 /**< If it's a file symlink. */
#define FILE_IS_SYMDIR 0x4000 /**< If it's a dir symlink for Windows. Not yet supported. */
@ -867,6 +877,12 @@ block_off_t disk_size(struct snapraid_disk* disk);
*/
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
/**
* Check the filesystem for errors.
* Return 0 if it's OK.
*/
int fs_check(struct snapraid_disk* disk);
/**
* Allocate a parity position for the specified file position.
*

6
cmdline/scan.c
View file

@ -317,6 +317,9 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil
block_state_set(block, BLOCK_STATE_DELETED);
}
/* mark the file as deleted */
file_flag_set(file, FILE_IS_DELETED);
/* insert it in the list of deleted blocks */
tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file);
}
@ -1772,6 +1775,9 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
tommy_list_foreach(&scanlist, (tommy_foreach_func*)free);
/* check the filesystem on all disks */
state_fscheck(state, "after scan");
if (is_diff) {
/* check for file difference */
if (!no_difference)

54
cmdline/state.c
View file

@ -1792,15 +1792,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
block_state_set(block, BLOCK_STATE_REP);
}
/* we must not overwrite existing blocks */
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Internal inconsistency in block existence!\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* set the parity association */
fs_allocate(disk, v_pos, file, v_idx);
@ -1958,7 +1949,10 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* if it's a run of deleted blocks */
/* allocate a fake deleted file */
deleted = file_alloc(state->block_size, "", v_count * state->block_size, 0, 0, 0, 0);
deleted = file_alloc(state->block_size, "<deleted>", v_count * state->block_size, 0, 0, 0, 0);
/* mark the file as deleted */
file_flag_set(deleted, FILE_IS_DELETED);
/* insert it in the list of deleted files */
tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
@ -1986,21 +1980,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
hash_invalid_set(block->hash);
}
/* we must not overwrite existing blocks */
if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos);
if (state->opt.skip_content_check) {
log_fatal("Overriding as used.\n");
} else {
decoding_error(path, f);
exit(EXIT_FAILURE);
}
/* LCOV_EXCL_STOP */
} else {
/* insert the block in the block array */
fs_allocate(disk, v_pos, deleted, v_idx);
}
/* insert the block in the block array */
fs_allocate(disk, v_pos, deleted, v_idx);
/* go to next block */
++v_pos;
@ -2424,6 +2405,9 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* LCOV_EXCL_STOP */
}
/* check the filesystem on all disks */
state_fscheck(state, "after read");
/* check that the stored parity size matches the loaded state */
if (blockmax != parity_allocated_size(state)) {
/* LCOV_EXCL_START */
@ -2467,6 +2451,9 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
/* blocks of all array */
blockmax = parity_allocated_size(state);
/* check the filesystem on all disks */
state_fscheck(state, "before write");
/* clear the info for unused blocks */
/* and get some other info */
info_oldest = 0; /* oldest time in info */
@ -3474,6 +3461,23 @@ void state_usage_print(struct snapraid_state* state)
}
}
void state_fscheck(struct snapraid_state* state, const char* ope)
{
tommy_node* i;
/* check the filesystem on all disks */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (fs_check(disk) != 0) {
/* LCOV_EXCL_START */
log_fatal("Internal inconsistency in filesystem for disk '%s' %s\n", disk->name, ope);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
void generate_configuration(const char* path)
{
struct snapraid_state state;

6
cmdline/state.h
View file

@ -372,6 +372,12 @@ static inline void state_usage_parity(struct snapraid_state* state, unsigned lev
*/
void state_usage_print(struct snapraid_state* state);
/**
* Check the filesystem on all disks.
* On error it aborts.
*/
void state_fscheck(struct snapraid_state* state, const char* ope);
/****************************************************************************/
/* misc */

9
cmdline/sync.c
View file

@ -1019,9 +1019,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (!io_error_on_this_block) {
/* for each disk, mark the blocks as processed */
for (j = 0; j < diskmax; ++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = fs_par2block_get(handle[j].disk, i);
struct snapraid_block* block;
if (!handle[j].disk)
continue;
block = fs_par2block_get(handle[j].disk, i);
if (block == BLOCK_EMPTY) {
/* nothing to do */