diff --git a/Makefile.am b/Makefile.am index c3d193b9..706d8bef 100644 --- a/Makefile.am +++ b/Makefile.am @@ -150,26 +150,27 @@ maintainer-clean-local: # Automatic testing -# Base options +# Common options # --test-skip-device # Is required to allow testing in the same disk # # --test-skip-self # It just makes the test a little faster -# +CHECKFLAGS_COMMON = --test-skip-device --test-skip-self --test-force-progress --no-warnings --test-parity-limit=3333333 + # --test-force-order-alpha # Ensures to process files always in the same order despites # the inode, physical location, and dir order assigned by the OS. -CHECKFLAGS_BASE = --test-skip-device --test-skip-self --test-force-order-alpha --test-force-progress --no-warnings +CHECKFLAGS_ALPHA = $(CHECKFLAGS_COMMON) --test-force-order-alpha # Physical offset options -CHECKFLAGS_PHYSICAL = --test-skip-device --test-skip-self --test-force-order-physical -q -q -q +CHECKFLAGS_PHYSICAL = $(CHECKFLAGS_COMMON) --test-force-order-physical -q -q -q # Verbose options -CHECKFLAGS_VERBOSE = $(CHECKFLAGS_BASE) -v -G +CHECKFLAGS_VERBOSE = $(CHECKFLAGS_ALPHA) -v -G # Quiet options -CHECKFLAGS = $(CHECKFLAGS_BASE) -q -q -q +CHECKFLAGS = $(CHECKFLAGS_ALPHA) -q -q -q # Number of files to create if !HAVE_MEMORY_CHECKER @@ -842,34 +843,34 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check #### RECOVER PARITY #### $(MSG) Delete the parity, fix and check with PAR1 - rm bench/parity + rm bench/parity* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Delete the parity and the 2-parity, fix and check with PAR2 - rm bench/parity - rm bench/2-parity + rm bench/parity* + rm bench/2-parity* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Delete the parity and a disk, fix and check with PAR2 - rm bench/parity + rm bench/parity* rm -r bench/disk2/b $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Delete the 2-parity and a disk, fix and check with PAR2 - rm bench/2-parity + rm bench/2-parity* rm -r bench/disk3/a $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Corrupt the parity, fix and check with PAR1 - $(TESTENV) ./mktest$(EXEEXT) write 1 100 10000 bench/parity + $(TESTENV) ./mktest$(EXEEXT) write 1 100 10000 bench/parity* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-expect-recoverable -c $(PAR1) check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) -a check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Corrupt the parity and the 2-parity, fix and check with PAR2 - $(TESTENV) ./mktest$(EXEEXT) write 2 100 10000 bench/parity - $(TESTENV) ./mktest$(EXEEXT) write 2 100 10000 bench/2-parity + $(TESTENV) ./mktest$(EXEEXT) write 2 100 10000 bench/parity* + $(TESTENV) ./mktest$(EXEEXT) write 2 100 10000 bench/2-parity* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-expect-recoverable -c $(PAR2) check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -a check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log @@ -898,14 +899,14 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Corrupt the parity and some files, fix and check with PAR2 - $(TESTENV) ./mktest$(EXEEXT) write 6 100 10000 bench/parity + $(TESTENV) ./mktest$(EXEEXT) write 6 100 10000 bench/parity* $(TESTENV) ./mktest$(EXEEXT) change 6 500 bench/disk1/a/* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-expect-unrecoverable -c $(PAR1) check -l test-fail-corrupt-parity.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-expect-recoverable -c $(PAR2) check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(MSG) Corrupt the 2-parity and some files, fix and check with PAR2 - $(TESTENV) ./mktest$(EXEEXT) write 7 100 10000 bench/2-parity + $(TESTENV) ./mktest$(EXEEXT) write 7 100 10000 bench/2-parity* $(TESTENV) ./mktest$(EXEEXT) change 7 500 bench/disk1/a/* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-expect-recoverable -c $(PAR1) check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-expect-recoverable -c $(PAR2) check -l test.log diff --git a/cmdline/check.c b/cmdline/check.c index 2de32ab8..7c2bc4f8 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -1947,7 +1947,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b } parity_ptr[l] = &parity[l]; - ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode); + ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode, state->opt.parity_limit_size); if (ret == -1) { /* LCOV_EXCL_START */ log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l)); @@ -1968,7 +1968,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b /* it may fail if the file doesn't exist, in this case we continue to check the files */ for (l = 0; l < state->level; ++l) { parity_ptr[l] = &parity[l]; - ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode); + ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode, state->opt.parity_limit_size); if (ret == -1) { msg_status("No accessible %s file, only files will be checked.\n", lev_name(l)); /* continue anyway */ diff --git a/cmdline/dry.c b/cmdline/dry.c index 635a60ce..49f83fcf 100644 --- a/cmdline/dry.c +++ b/cmdline/dry.c @@ -417,7 +417,7 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t /* open the file for reading */ /* it may fail if the file doesn't exist, in this case we continue to dry the files */ for (l = 0; l < state->level; ++l) { - ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode); + ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode, state->opt.parity_limit_size); if (ret == -1) { /* LCOV_EXCL_START */ log_fatal("WARNING! Without an accessible %s file, it isn't possible to dry.\n", lev_name(l)); diff --git a/cmdline/parity.c b/cmdline/parity.c index 3b724192..2c91ac43 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -23,6 +23,12 @@ #include "parity.h" #include "handle.h" +/** + * Pseudo random limits for parity + */ +#define PARITY_LIMIT(size,split,level) \ + size ? size + (123562341 + split * 634542351 + level * 983491341) % size : 0 + /****************************************************************************/ /* parity */ @@ -178,7 +184,7 @@ void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size) *out_size = size; } -int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode) +int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode, uint64_t limit_size) { unsigned s; @@ -194,6 +200,7 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p pathcpy(split->path, sizeof(split->path), parity->split_map[s].path); split->f = -1; split->size = parity->split_map[s].size; + split->limit_size = PARITY_LIMIT(limit_size, s, level); ++handle->split_mac; /* opening in sequential mode in Windows */ @@ -248,6 +255,10 @@ static int parity_handle_grow(struct snapraid_split_handle* split, data_off_t si { int ret; + /* simulate a failure for testing limits */ + if (split->limit_size != 0 && size > (data_off_t)split->limit_size) + return -1; + #if HAVE_FALLOCATE if (!skip_fallocate) { /* @@ -500,7 +511,7 @@ int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* return 0; } -int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode) +int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode, uint64_t limit_size) { unsigned s; @@ -516,6 +527,7 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par pathcpy(split->path, sizeof(split->path), parity->split_map[s].path); split->f = -1; split->size = parity->split_map[s].size; + split->limit_size = PARITY_LIMIT(limit_size, s, level); ++handle->split_mac; /* open for read */ diff --git a/cmdline/parity.h b/cmdline/parity.h index b6eab8b7..8c2ea093 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -35,6 +35,12 @@ struct snapraid_split_handle { * Note that this value CANNOT be PARITY_SIZE_INVALID. */ uint64_t size; + + /** + * Artificial size limit for testing. + * 0 means unlimited. + */ + uint64_t limit_size; }; struct snapraid_parity_handle { @@ -78,12 +84,11 @@ int parity_is_invalid(struct snapraid_state* state); */ void parity_overflow(struct snapraid_state* state, data_off_t size); - /** * Create the parity file. * \param out_size Return the size of the parity file. */ -int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode); +int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode, uint64_t limit_size); /** * Change the parity size. @@ -101,7 +106,7 @@ void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size); /** * Open an already existing parity file. */ -int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode); +int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode, uint64_t limit_size); /** * Flush the parity file in the disk. diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 7875b55f..802ec0c9 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -854,7 +854,7 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan) /* open the file for reading */ for (l = 0; l < state->level; ++l) { - ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode); + ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode, state->opt.parity_limit_size); if (ret == -1) { /* LCOV_EXCL_START */ log_fatal("WARNING! Without an accessible %s file, it isn't possible to scrub.\n", lev_name(l)); diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 999aa9ee..d7bbc16b 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -290,6 +290,7 @@ void config(char* conf, size_t conf_size, const char* argv0) #define OPT_TEST_IO_ADVISE_DISCARD_8 298 #define OPT_TEST_IO_ADVISE_DISCARD_32 299 #define OPT_TEST_IO_ADVISE_DIRECT 300 +#define OPT_TEST_PARITY_LIMIT 301 #if HAVE_GETOPT_LONG struct option long_options[] = { @@ -447,6 +448,9 @@ struct option long_options[] = { /* Set the io advise to direct */ { "test-io-advise-direct", 0, 0, OPT_TEST_IO_ADVISE_DIRECT }, + /* Set an artificial parity limit */ + { "test-parity-limit", 1, 0, OPT_TEST_PARITY_LIMIT }, + { 0, 0, 0, 0 } }; #endif @@ -867,6 +871,9 @@ int main(int argc, char* argv[]) case OPT_TEST_IO_ADVISE_DIRECT : opt.file_mode = ADVISE_DIRECT; break; + case OPT_TEST_PARITY_LIMIT : + opt.parity_limit_size = atoll(optarg); + break; default : /* LCOV_EXCL_START */ log_fatal("Unknown option '%c'\n", (char)c); diff --git a/cmdline/state.h b/cmdline/state.h index b5f848d7..92cf181b 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -114,6 +114,7 @@ struct snapraid_option { unsigned io_cache; /**< Number of IO buffers to use. 0 for default. */ int auto_conf; /**< Allow to run without configuration file. */ int force_stats; /**< Force stats print during process. */ + uint64_t parity_limit_size; /**< Test limit for parity files. */ }; struct snapraid_state { diff --git a/cmdline/sync.c b/cmdline/sync.c index 960d29dd..09000501 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1448,7 +1448,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t block_off_t parityblocks; /* create the file and open for writing */ - ret = parity_create(&parity_handle[l], &state->parity[l], l, state->file_mode); + ret = parity_create(&parity_handle[l], &state->parity[l], l, state->file_mode, state->opt.parity_limit_size); if (ret == -1) { /* LCOV_EXCL_START */ log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l)); diff --git a/test/test-par1.conf b/test/test-par1.conf index b0e48400..93d45595 100644 --- a/test/test-par1.conf +++ b/test/test-par1.conf @@ -1,6 +1,6 @@ # Test configuration file blocksize 1 -parity bench/parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 content bench/content content bench/1-content disk disk1 bench/disk1/ diff --git a/test/test-par2.conf b/test/test-par2.conf index 219aa26e..917bab17 100644 --- a/test/test-par2.conf +++ b/test/test-par2.conf @@ -1,6 +1,6 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par3.conf b/test/test-par3.conf index 5c726d40..af1af7c1 100644 --- a/test/test-par3.conf +++ b/test/test-par3.conf @@ -1,7 +1,7 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par4.conf b/test/test-par4.conf index 9bf51d83..bba81dde 100644 --- a/test/test-par4.conf +++ b/test/test-par4.conf @@ -1,8 +1,8 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity -4-parity bench/4-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 +4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par5.conf b/test/test-par5.conf index 2a698ff9..d3bf8e31 100644 --- a/test/test-par5.conf +++ b/test/test-par5.conf @@ -1,9 +1,9 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity -4-parity bench/4-parity -5-parity bench/5-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 +4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 +5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par6-hole.conf b/test/test-par6-hole.conf index e62c6220..00d0945e 100644 --- a/test/test-par6-hole.conf +++ b/test/test-par6-hole.conf @@ -1,10 +1,10 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity -4-parity bench/4-parity -5-parity bench/5-parity -6-parity bench/6-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 +4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 +5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 +6-parity bench/6-parity.0,bench/6-parity.1,bench/6-parity.2,bench/6-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par6-noaccess.conf b/test/test-par6-noaccess.conf index 3cfa56bd..0f38aee2 100644 --- a/test/test-par6-noaccess.conf +++ b/test/test-par6-noaccess.conf @@ -1,10 +1,10 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity -4-parity bench/4-parity -5-parity bench/5-parity -6-parity bench/no-access/6-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 +4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 +5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 +6-parity bench/no-access/6-parity.0,bench/no-access/6-parity.1,bench/no-access/6-parity.2,bench/no-access/6-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par6-rename.conf b/test/test-par6-rename.conf index cc7f6ce9..df9c80a4 100644 --- a/test/test-par6-rename.conf +++ b/test/test-par6-rename.conf @@ -1,10 +1,10 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity -4-parity bench/4-parity -5-parity bench/5-parity -6-parity bench/6-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 +4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 +5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 +6-parity bench/6-parity.0,bench/6-parity.1,bench/6-parity.2,bench/6-parity.3 content bench/content content bench/1-content content bench/2-content diff --git a/test/test-par6.conf b/test/test-par6.conf index c92e1dbd..15112bc0 100644 --- a/test/test-par6.conf +++ b/test/test-par6.conf @@ -1,10 +1,10 @@ blocksize 1 -parity bench/parity -2-parity bench/2-parity -3-parity bench/3-parity -4-parity bench/4-parity -5-parity bench/5-parity -6-parity bench/6-parity +parity bench/parity.0,bench/parity.1,bench/parity.2,bench/parity.3 +2-parity bench/2-parity.0,bench/2-parity.1,bench/2-parity.2,bench/2-parity.3 +3-parity bench/3-parity.0,bench/3-parity.1,bench/3-parity.2,bench/3-parity.3 +4-parity bench/4-parity.0,bench/4-parity.1,bench/4-parity.2,bench/4-parity.3 +5-parity bench/5-parity.0,bench/5-parity.1,bench/5-parity.2,bench/5-parity.3 +6-parity bench/6-parity.0,bench/6-parity.1,bench/6-parity.2,bench/6-parity.3 content bench/content content bench/1-content content bench/2-content