rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fixes a bug in check when detecting garbage data over the expected end of a file.

Browse source 
More extensive tests to detect the append/truncate conditions.
This commit is contained in:
Andrea Mazzoleni 2011-04-28 21:38:25 +02:00
parent 5baef32fb9
commit a9255683f4
8 changed files with 204 additions and 42 deletions
Download patch file Download diff file Expand all files Collapse all files

1
HISTORY
View file

@ -3,6 +3,7 @@ SnapRAID HISTORY
1.1 2011/5
==========
* Fixed a bug in check when detecting garbage data over the expected end of the file.
* Added RAID6 support and allow to specify multiple content files.
* Added filtering include logic, where anything not explicitely included is excluded.
* Changed the default checksum to Murmur3 hash. It's a lot faster than MD5.

55
Makefile.am
View file

@ -178,38 +178,71 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Corrupt the parity, fix and check with raid5
$(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/parity
$(TESTENV) ./mktest$(EXEEXT) damage 1 100 10000 test/parity
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid5.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid5.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Corrupt the parity and the q-parity, fix and check with raid6
$(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/parity
$(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/q-parity
$(TESTENV) ./mktest$(EXEEXT) damage 2 100 10000 test/parity
$(TESTENV) ./mktest$(EXEEXT) damage 2 100 10000 test/q-parity
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Corrupt some files, fix and check with raid5
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/b*
$(TESTENV) ./mktest$(EXEEXT) damage 3 1 500 test/disk2/b*
$(TESTENV) ./mktest$(EXEEXT) append 3 500 test/disk2/c*
$(TESTENV) ./mktest$(EXEEXT) truncate 3 500 test/disk2/d*
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid5.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid5.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Corrupt some files, fix and check with raid6
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk3/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk3/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk2/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk3/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk2/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk3/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) append 4 500 test/disk2/_b/_a/*
$(TESTENV) ./mktest$(EXEEXT) append 4 500 test/disk3/_b/_a/*
$(TESTENV) ./mktest$(EXEEXT) truncate 4 500 test/disk2/_b/_b/*
$(TESTENV) ./mktest$(EXEEXT) truncate 4 500 test/disk3/_b/_b/*
$(TESTENV) ./snapraid$(EXEEXT) -U -c $(srcdir)/test-raid5.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Another time with different values
$(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk2/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk3/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) append 5 500 test/disk2/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) truncate 5 500 test/disk3/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk2/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk3/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) append 5 1500 test/disk2/_b/_a/*
$(TESTENV) ./mktest$(EXEEXT) append 5 1500 test/disk3/_b/_a/*
$(TESTENV) ./mktest$(EXEEXT) truncate 5 1500 test/disk2/_b/_b/*
$(TESTENV) ./mktest$(EXEEXT) truncate 5 1500 test/disk3/_b/_b/*
$(TESTENV) ./snapraid$(EXEEXT) -U -c $(srcdir)/test-raid5.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Corrupt the parity and some files, fix and check with raid6
$(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/parity
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk1/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk1/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 6 100 10000 test/parity
$(TESTENV) ./mktest$(EXEEXT) damage 6 1 500 test/disk1/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 6 1 500 test/disk1/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) append 6 500 test/disk1/_b/_a/*
$(TESTENV) ./mktest$(EXEEXT) truncate 6 500 test/disk1/_b/_b/*
$(TESTENV) ./snapraid$(EXEEXT) -U -c $(srcdir)/test-raid5.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Corrupt the q-parity and some files, fix and check with raid6
$(TESTENV) ./mktest$(EXEEXT) damage 7 100 10000 test/q-parity
$(TESTENV) ./mktest$(EXEEXT) damage 7 1 500 test/disk1/_a/_a/*
$(TESTENV) ./mktest$(EXEEXT) damage 7 1 500 test/disk1/_a/_b/*
$(TESTENV) ./mktest$(EXEEXT) append 7 500 test/disk1/_b/_a/*
$(TESTENV) ./mktest$(EXEEXT) truncate 7 500 test/disk1/_b/_b/*
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid5.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check
echo --- Sync after all the fixes
$(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check

83
check.c
View file

@ -33,6 +33,34 @@ struct failed_struct {
struct snapraid_handle* handle;
};
/**
* Checks if a block hash matches the specified buffer.
*/
static int blockcmp(struct snapraid_state* state, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero)
{
unsigned char hash[HASH_SIZE];
unsigned size;
size = block_file_size(block, state->block_size);
/* now compute the hash of the valid part */
memhash(state->hash, hash, buffer, size);
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
return -1;
}
/* compare to the end of the block */
if (size < state->block_size) {
if (memcmp(buffer + size, buffer_zero + size, state->block_size - size) != 0) {
return -1;
}
}
return 0;
}
/**
* Repair errors.
* Returns <0 if failure for missing stratey, >0 if data is wrong and we cannot rebuild correctly, 0 on success.
@ -40,8 +68,6 @@ struct failed_struct {
*/
static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, struct failed_struct* failed, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
{
unsigned char hash[HASH_SIZE];
unsigned size;
unsigned j;
int error = 0;
@ -60,15 +86,8 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
raid5_recov_data(buffer, diskmax, state->block_size, failed[0].index);
for(j=0;j<1;++j) {
size = block_file_size(failed[j].block, state->block_size);
/* now compute the hash */
memhash(state->hash, hash, buffer[failed[j].index], size);
/* compare the hash */
if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) {
break;
}
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
}
if (j==1) {
@ -89,15 +108,8 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
raid6_recov_datap(buffer, diskmax, state->block_size, failed[0].index, buffer_zero);
for(j=0;j<1;++j) {
size = block_file_size(failed[j].block, state->block_size);
/* now compute the hash */
memhash(state->hash, hash, buffer[failed[j].index], size);
/* compare the hash */
if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) {
break;
}
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
}
if (j==1) {
@ -120,15 +132,8 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
raid6_recov_2data(buffer, diskmax, state->block_size, failed[0].index, failed[1].index, buffer_zero);
for(j=0;j<2;++j) {
size = block_file_size(failed[j].block, state->block_size);
/* now compute the hash */
memhash(state->hash, hash, buffer[failed[j].index], size);
/* compare the hash */
if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) {
break;
}
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
}
if (j==2) {
@ -279,7 +284,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
if (fix) {
/* if fixing, create the file and open for writing */
/* if fixing, create the file, open for writing and resize if required */
ret = handle_create(&handle[j], block->file);
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
@ -287,6 +292,13 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
++unrecoverable_error;
goto bail;
}
/* notify the warning condition of a file larger than expected */
if (ret > 0) {
error += ret;
/* this is always a recovered error */
recovered_error += ret;
}
} else {
/* if checking, open the file for reading */
ret = handle_open(&handle[j], block->file);
@ -303,6 +315,17 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
++error;
continue;
}
/* check if it's a larger file, but not if already notified */
if (!block->file->is_larger && handle[j].st.st_size > block->file->size) {
fprintf(stderr, "File '%s' is larger than expected.\n", handle->path);
/* if fragmented, it may be reopened, so store the notification */
block->file->is_larger = 1;
/* this is always a recoverable error */
++error;
}
}
read_size = handle_read(&handle[j], block, buffer[j], state->block_size);

1
elem.c
View file

@ -224,6 +224,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t
file->inode = inode;
file->is_present = 0;
file->is_excluded = 0;
file->is_larger = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
/* set the back pointer */

1
elem.h
View file

@ -90,6 +90,7 @@ struct snapraid_file {
uint64_t inode; /**< Inode. */
int is_present; /**< If it's seen as present. */
int is_excluded; /**< If it's an excluded file from the processing. */
int is_larger; /**< If a larger file was already detected. */
/* nodes for data structures */
tommy_node nodelist;
tommy_hashdyn_node nodeset;

6
handle.c
View file

@ -164,6 +164,12 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file)
}
#endif
/* return the warning condition of a larger file */
if (handle->st.st_size > file->size) {
fprintf(stderr, "File '%s' truncated to size %llu.\n", handle->path, file->size);
return 1;
}
return 0;
}

1
handle.h
View file

@ -37,6 +37,7 @@ int handle_close_if_different(struct snapraid_handle* handle, struct snapraid_fi
/**
* Creates a file.
* The file is created if missing, and opening with write access.
* If the file is larger than expected, the value 1 is returned, and the file is truncated to the correct size.
*/
int handle_create(struct snapraid_handle* handle, struct snapraid_file* file);

98
mktest.c
View file

@ -123,6 +123,9 @@ void damage(const char* path, int size)
off_t start;
int count;
if (!size)
return;
f = fopen(path, "r+b");
if (!f) {
fprintf(stderr, "Error writing %s\n", path);
@ -135,7 +138,10 @@ void damage(const char* path, int size)
}
/* start at random position inside the file */
start = rnd(st.st_size);
if (st.st_size)
start = rnd(st.st_size);
else
start = 0;
if (fseek(f, start, SEEK_SET) != 0) {
fprintf(stderr, "Error seeking %s\n", path);
exit(EXIT_FAILURE);
@ -151,12 +157,74 @@ void damage(const char* path, int size)
fclose(f);
}
void append(const char* path, int size)
{
FILE* f;
int count;
if (!size)
return;
f = fopen(path, "ab");
if (!f) {
fprintf(stderr, "Error appending %s\n", path);
exit(EXIT_FAILURE);
}
/* write garbage, in case also over the end */
count = rnd(size);
while (count) {
fputc(rnd(256), f);
--count;
}
fclose(f);
}
void truncat(const char* path, int size)
{
FILE* f;
struct stat st;
off_t start;
int count;
if (!size)
return;
f = fopen(path, "r+b");
if (!f) {
fprintf(stderr, "Error writing %s\n", path);
exit(EXIT_FAILURE);
}
if (fstat(fileno(f), &st) != 0) {
fprintf(stderr, "Error accessing %s\n", path);
exit(EXIT_FAILURE);
}
/* truncate at random position inside the file */
count = rnd(size);
if (count > st.st_size)
count = st.st_size;
start = st.st_size - count;
if (ftruncate(fileno(f), start) != 0) {
fprintf(stderr, "Error truncating %s\n", path);
exit(EXIT_FAILURE);
}
fclose(f);
}
void help(void)
{
printf("Test for " PACKAGE " v" VERSION " by Andrea Mazzoleni, " PACKAGE_URL "\n");
printf("Usage:\n");
printf("\tmktest generate SEED DISK_NUM FILE_NUM FILE_SIZE\n");
printf("\tmktest damage SEED FAIL_NUM FAIL_SIZE FILE\n");
printf("\tmktest append SEED FAIL_SIZE FILE\n");
printf("\tmktest truncate SEED FAIL_SIZE FILE\n");
}
int main(int argc, char* argv[])
@ -208,6 +276,34 @@ int main(int argc, char* argv[])
damage(argv[i], rnd(size));
}
}
} else if (strcmp(argv[1], "append") == 0) {
int size;
if (argc < 5) {
help();
exit(EXIT_FAILURE);
}
seed = atoi(argv[2]);
size = atoi(argv[3]);
for(i=4;i<argc;++i) {
append(argv[i], rnd(size));
}
} else if (strcmp(argv[1], "truncate") == 0) {
int size;
if (argc < 5) {
help();
exit(EXIT_FAILURE);
}
seed = atoi(argv[2]);
size = atoi(argv[3]);
for(i=4;i<argc;++i) {
truncat(argv[i], rnd(size));
}
} else {
help();
exit(EXIT_FAILURE);