diff --git a/HISTORY b/HISTORY index 360fa5db..9331190b 100644 --- a/HISTORY +++ b/HISTORY @@ -3,6 +3,7 @@ SnapRAID HISTORY 1.1 2011/5 ========== +* Fixed a bug in check when detecting garbage data over the expected end of the file. * Added RAID6 support and allow to specify multiple content files. * Added filtering include logic, where anything not explicitely included is excluded. * Changed the default checksum to Murmur3 hash. It's a lot faster than MD5. diff --git a/Makefile.am b/Makefile.am index 8839af45..e8929a59 100644 --- a/Makefile.am +++ b/Makefile.am @@ -178,38 +178,71 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check echo --- Corrupt the parity, fix and check with raid5 - $(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/parity + $(TESTENV) ./mktest$(EXEEXT) damage 1 100 10000 test/parity $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid5.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid5.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check echo --- Corrupt the parity and the q-parity, fix and check with raid6 - $(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/parity - $(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/q-parity + $(TESTENV) ./mktest$(EXEEXT) damage 2 100 10000 test/parity + $(TESTENV) ./mktest$(EXEEXT) damage 2 100 10000 test/q-parity $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check echo --- Corrupt some files, fix and check with raid5 - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/b* + $(TESTENV) ./mktest$(EXEEXT) damage 3 1 500 test/disk2/b* + $(TESTENV) ./mktest$(EXEEXT) append 3 500 test/disk2/c* + $(TESTENV) ./mktest$(EXEEXT) truncate 3 500 test/disk2/d* $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid5.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid5.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check echo --- Corrupt some files, fix and check with raid6 - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/_a/_a/* - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk3/_a/_a/* - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/_a/_b/* - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk3/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk2/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk3/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk2/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) damage 4 1 500 test/disk3/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) append 4 500 test/disk2/_b/_a/* + $(TESTENV) ./mktest$(EXEEXT) append 4 500 test/disk3/_b/_a/* + $(TESTENV) ./mktest$(EXEEXT) truncate 4 500 test/disk2/_b/_b/* + $(TESTENV) ./mktest$(EXEEXT) truncate 4 500 test/disk3/_b/_b/* + $(TESTENV) ./snapraid$(EXEEXT) -U -c $(srcdir)/test-raid5.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check + echo --- Another time with different values + $(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk2/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk3/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) append 5 500 test/disk2/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) truncate 5 500 test/disk3/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk2/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) damage 5 2 50 test/disk3/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) append 5 1500 test/disk2/_b/_a/* + $(TESTENV) ./mktest$(EXEEXT) append 5 1500 test/disk3/_b/_a/* + $(TESTENV) ./mktest$(EXEEXT) truncate 5 1500 test/disk2/_b/_b/* + $(TESTENV) ./mktest$(EXEEXT) truncate 5 1500 test/disk3/_b/_b/* $(TESTENV) ./snapraid$(EXEEXT) -U -c $(srcdir)/test-raid5.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check echo --- Corrupt the parity and some files, fix and check with raid6 - $(TESTENV) ./mktest$(EXEEXT) damage 1 50 10000 test/parity - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk1/_a/_a/* - $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk1/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 6 100 10000 test/parity + $(TESTENV) ./mktest$(EXEEXT) damage 6 1 500 test/disk1/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 6 1 500 test/disk1/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) append 6 500 test/disk1/_b/_a/* + $(TESTENV) ./mktest$(EXEEXT) truncate 6 500 test/disk1/_b/_b/* $(TESTENV) ./snapraid$(EXEEXT) -U -c $(srcdir)/test-raid5.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check + echo --- Corrupt the q-parity and some files, fix and check with raid6 + $(TESTENV) ./mktest$(EXEEXT) damage 7 100 10000 test/q-parity + $(TESTENV) ./mktest$(EXEEXT) damage 7 1 500 test/disk1/_a/_a/* + $(TESTENV) ./mktest$(EXEEXT) damage 7 1 500 test/disk1/_a/_b/* + $(TESTENV) ./mktest$(EXEEXT) append 7 500 test/disk1/_b/_a/* + $(TESTENV) ./mktest$(EXEEXT) truncate 7 500 test/disk1/_b/_b/* + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid5.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test-raid6.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf fix 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check echo --- Sync after all the fixes $(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test-raid6.conf check diff --git a/check.c b/check.c index 29afcc81..ff93e17e 100644 --- a/check.c +++ b/check.c @@ -33,6 +33,34 @@ struct failed_struct { struct snapraid_handle* handle; }; +/** + * Checks if a block hash matches the specified buffer. + */ +static int blockcmp(struct snapraid_state* state, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero) +{ + unsigned char hash[HASH_SIZE]; + unsigned size; + + size = block_file_size(block, state->block_size); + + /* now compute the hash of the valid part */ + memhash(state->hash, hash, buffer, size); + + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_SIZE) != 0) { + return -1; + } + + /* compare to the end of the block */ + if (size < state->block_size) { + if (memcmp(buffer + size, buffer_zero + size, state->block_size - size) != 0) { + return -1; + } + } + + return 0; +} + /** * Repair errors. * Returns <0 if failure for missing stratey, >0 if data is wrong and we cannot rebuild correctly, 0 on success. @@ -40,8 +68,6 @@ struct failed_struct { */ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, struct failed_struct* failed, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero) { - unsigned char hash[HASH_SIZE]; - unsigned size; unsigned j; int error = 0; @@ -60,15 +86,8 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st raid5_recov_data(buffer, diskmax, state->block_size, failed[0].index); for(j=0;j<1;++j) { - size = block_file_size(failed[j].block, state->block_size); - - /* now compute the hash */ - memhash(state->hash, hash, buffer[failed[j].index], size); - - /* compare the hash */ - if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) { - break; - } + if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0) + break; } if (j==1) { @@ -89,15 +108,8 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st raid6_recov_datap(buffer, diskmax, state->block_size, failed[0].index, buffer_zero); for(j=0;j<1;++j) { - size = block_file_size(failed[j].block, state->block_size); - - /* now compute the hash */ - memhash(state->hash, hash, buffer[failed[j].index], size); - - /* compare the hash */ - if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) { - break; - } + if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0) + break; } if (j==1) { @@ -120,15 +132,8 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st raid6_recov_2data(buffer, diskmax, state->block_size, failed[0].index, failed[1].index, buffer_zero); for(j=0;j<2;++j) { - size = block_file_size(failed[j].block, state->block_size); - - /* now compute the hash */ - memhash(state->hash, hash, buffer[failed[j].index], size); - - /* compare the hash */ - if (memcmp(hash, failed[j].block->hash, HASH_SIZE) != 0) { - break; - } + if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0) + break; } if (j==2) { @@ -279,7 +284,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } if (fix) { - /* if fixing, create the file and open for writing */ + /* if fixing, create the file, open for writing and resize if required */ ret = handle_create(&handle[j], block->file); if (ret == -1) { fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); @@ -287,6 +292,13 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity ++unrecoverable_error; goto bail; } + + /* notify the warning condition of a file larger than expected */ + if (ret > 0) { + error += ret; + /* this is always a recovered error */ + recovered_error += ret; + } } else { /* if checking, open the file for reading */ ret = handle_open(&handle[j], block->file); @@ -303,6 +315,17 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity ++error; continue; } + + /* check if it's a larger file, but not if already notified */ + if (!block->file->is_larger && handle[j].st.st_size > block->file->size) { + fprintf(stderr, "File '%s' is larger than expected.\n", handle->path); + + /* if fragmented, it may be reopened, so store the notification */ + block->file->is_larger = 1; + + /* this is always a recoverable error */ + ++error; + } } read_size = handle_read(&handle[j], block, buffer[j], state->block_size); diff --git a/elem.c b/elem.c index b3e3f4f1..ed92c5a1 100644 --- a/elem.c +++ b/elem.c @@ -224,6 +224,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t file->inode = inode; file->is_present = 0; file->is_excluded = 0; + file->is_larger = 0; file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); /* set the back pointer */ diff --git a/elem.h b/elem.h index 327e9b78..30758910 100644 --- a/elem.h +++ b/elem.h @@ -90,6 +90,7 @@ struct snapraid_file { uint64_t inode; /**< Inode. */ int is_present; /**< If it's seen as present. */ int is_excluded; /**< If it's an excluded file from the processing. */ + int is_larger; /**< If a larger file was already detected. */ /* nodes for data structures */ tommy_node nodelist; tommy_hashdyn_node nodeset; diff --git a/handle.c b/handle.c index b6344189..1b5d3ef6 100644 --- a/handle.c +++ b/handle.c @@ -164,6 +164,12 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file) } #endif + /* return the warning condition of a larger file */ + if (handle->st.st_size > file->size) { + fprintf(stderr, "File '%s' truncated to size %llu.\n", handle->path, file->size); + return 1; + } + return 0; } diff --git a/handle.h b/handle.h index ec000e0b..e2ee0155 100644 --- a/handle.h +++ b/handle.h @@ -37,6 +37,7 @@ int handle_close_if_different(struct snapraid_handle* handle, struct snapraid_fi /** * Creates a file. * The file is created if missing, and opening with write access. + * If the file is larger than expected, the value 1 is returned, and the file is truncated to the correct size. */ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file); diff --git a/mktest.c b/mktest.c index 2897279c..ecc04ed8 100644 --- a/mktest.c +++ b/mktest.c @@ -123,6 +123,9 @@ void damage(const char* path, int size) off_t start; int count; + if (!size) + return; + f = fopen(path, "r+b"); if (!f) { fprintf(stderr, "Error writing %s\n", path); @@ -135,7 +138,10 @@ void damage(const char* path, int size) } /* start at random position inside the file */ - start = rnd(st.st_size); + if (st.st_size) + start = rnd(st.st_size); + else + start = 0; if (fseek(f, start, SEEK_SET) != 0) { fprintf(stderr, "Error seeking %s\n", path); exit(EXIT_FAILURE); @@ -151,12 +157,74 @@ void damage(const char* path, int size) fclose(f); } +void append(const char* path, int size) +{ + FILE* f; + int count; + + if (!size) + return; + + f = fopen(path, "ab"); + if (!f) { + fprintf(stderr, "Error appending %s\n", path); + exit(EXIT_FAILURE); + } + + /* write garbage, in case also over the end */ + count = rnd(size); + while (count) { + fputc(rnd(256), f); + --count; + } + + fclose(f); +} + +void truncat(const char* path, int size) +{ + FILE* f; + struct stat st; + off_t start; + int count; + + if (!size) + return; + + f = fopen(path, "r+b"); + if (!f) { + fprintf(stderr, "Error writing %s\n", path); + exit(EXIT_FAILURE); + } + + if (fstat(fileno(f), &st) != 0) { + fprintf(stderr, "Error accessing %s\n", path); + exit(EXIT_FAILURE); + } + + /* truncate at random position inside the file */ + count = rnd(size); + if (count > st.st_size) + count = st.st_size; + start = st.st_size - count; + + if (ftruncate(fileno(f), start) != 0) { + fprintf(stderr, "Error truncating %s\n", path); + exit(EXIT_FAILURE); + } + + fclose(f); +} + + void help(void) { printf("Test for " PACKAGE " v" VERSION " by Andrea Mazzoleni, " PACKAGE_URL "\n"); printf("Usage:\n"); printf("\tmktest generate SEED DISK_NUM FILE_NUM FILE_SIZE\n"); printf("\tmktest damage SEED FAIL_NUM FAIL_SIZE FILE\n"); + printf("\tmktest append SEED FAIL_SIZE FILE\n"); + printf("\tmktest truncate SEED FAIL_SIZE FILE\n"); } int main(int argc, char* argv[]) @@ -208,6 +276,34 @@ int main(int argc, char* argv[]) damage(argv[i], rnd(size)); } } + } else if (strcmp(argv[1], "append") == 0) { + int size; + + if (argc < 5) { + help(); + exit(EXIT_FAILURE); + } + + seed = atoi(argv[2]); + size = atoi(argv[3]); + + for(i=4;i