rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactor the parity size detection before sync.

Browse source 
Now we resize the parity only after checking the size of all
the parity file.
This commit is contained in:
Andrea Mazzoleni 2015-02-01 18:48:50 +01:00
parent 6844fb7527
commit ad57aafd8f
11 changed files with 112 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

2
cmdline/check.c
View file

@ -1855,7 +1855,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
printf("Initializing...\n");
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
size = blockmax * (data_off_t)state->block_size;
if (blockstart > blockmax) {

2
cmdline/dry.c
View file

@ -187,7 +187,7 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t
printf("Drying...\n");
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
if (blockstart > blockmax) {
/* LCOV_EXCL_START */

15
cmdline/elem.h
View file

@ -544,7 +544,8 @@ static inline int block_has_updated_hash(const struct snapraid_block* block)
}
/**
* Checks if the specified block has an a past hash.
* Checks if the specified block has a past hash,
* i.e. the hash of the data that it's now overwritten or lost.
*
* Note that EMPTY / BLK / REP return 0.
*/
@ -637,6 +638,18 @@ static inline int block_has_invalid_parity(const struct snapraid_block* block)
|| state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
}
/**
* Checks if the block is part of a file with valid parity.
*
* Note that anything different than BLK return 0.
*/
static inline int block_has_file_and_valid_parity(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK;
}
/**
* Gets the relative position of a block inside the file.
*/

33
cmdline/parity.c
View file

@ -26,7 +26,34 @@
/****************************************************************************/
/* parity */
block_off_t parity_size(struct snapraid_state* state)
block_off_t parity_allocated_size(struct snapraid_state* state)
{
block_off_t parity_block;
tommy_node* i;
/* compute the size of the parity file */
parity_block = 0;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
/* start from the declared size */
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
/* decrease the block until an allocated one, but part of a file */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1)))
--block;
/* get the highest value */
if (block > parity_block)
parity_block = block;
}
return parity_block;
}
block_off_t parity_used_size(struct snapraid_state* state)
{
block_off_t parity_block;
tommy_node* i;
@ -40,9 +67,7 @@ block_off_t parity_size(struct snapraid_state* state)
block_off_t block = tommy_arrayblk_size(&disk->blockarr);
/* decrease the block until an used one */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1)))
while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1)))
--block;
/* get the highest value */

20
cmdline/parity.h
View file

@ -29,9 +29,25 @@ struct snapraid_parity_handle {
};
/**
* Computes the size of the parity data in number of blocks.
* Computes the size of the allocated parity data in number of blocks.
*
* This includes parity blocks not yet written and still invalid.
*/
block_off_t parity_size(struct snapraid_state* state);
block_off_t parity_allocated_size(struct snapraid_state* state);
/**
* Computes the size of the used parity data in number of blocks.
*
* This includes only parity blocks used for files, not counting
* potential invalid parity at the end.
*
* If the array is fully synched there is no difference between
* parity_allocate_size() and parity_used_size().
* But if the sync is interrupted, the parity_used_size() returns
* the position of the latest BLK block, ignoring CHG, REL and DELETED ones,
* because their parity may be still not even written in the parity file.
*/
block_off_t parity_used_size(struct snapraid_state* state);
/**
* Reports all the files outside the specified parity size.

2
cmdline/rehash.c
View file

@ -33,7 +33,7 @@ void state_rehash(struct snapraid_state* state)
block_off_t blockmax;
block_off_t i;
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
/* check if a rehash is already in progress */
if (state->prevhash != HASH_UNDEFINED) {

2
cmdline/scrub.c
View file

@ -567,7 +567,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
printf("Initializing...\n");
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
if (state->opt.force_scrub_even) {
/* no limit */

31
cmdline/state.c
View file

@ -99,7 +99,6 @@ void state_init(struct snapraid_state* state)
state->pool_device = 0;
state->lockfile[0] = 0;
state->level = 1; /* default is the lowest protection */
state->loaded_paritymax = 0;
state->clear_past_hash = 0;
state->no_conf = 0;
@ -1338,7 +1337,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
struct snapraid_file* file;
block_off_t blockidx;
block_off_t blockmax;
block_off_t paritymax;
unsigned line;
unsigned count_file;
unsigned count_hardlink;
@ -1351,7 +1349,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
line = 1;
blockidx = 0;
blockmax = 0;
paritymax = 0;
count_file = 0;
count_hardlink = 0;
count_symlink = 0;
@ -1420,10 +1417,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
switch (tag[0]) {
case 'b' :
block_state_set(block, BLOCK_STATE_BLK);
/* keep track of the required parity size */
if (v_pos + 1 > paritymax)
paritymax = v_pos + 1;
break;
case 'n' :
/* deprecated NEW blocks are converted to CHG ones */
@ -2383,7 +2376,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
/* check that the stored parity size matches the loaded state */
if (blockmax != parity_size(state)) {
if (blockmax != parity_allocated_size(state)) {
/* LCOV_EXCL_START */
fprintf(stderr, "Internal inconsistency in parity size in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
@ -2410,9 +2403,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
}
/* set the required parity size */
state->loaded_paritymax = paritymax;
if (state->opt.verbose) {
printf("%8u files\n", count_file);
printf("%8u hardlinks\n", count_hardlink);
@ -2439,7 +2429,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
count_dir = 0;
/* blocks of all array */
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
/* clear the info for unused blocks */
/* and get some other info */
@ -2883,7 +2873,6 @@ static void decoding_error(const char* path, STREAM* f)
static void state_read_binary(struct snapraid_state* state, const char* path, STREAM* f)
{
block_off_t blockmax;
block_off_t paritymax;
unsigned count_file;
unsigned count_hardlink;
unsigned count_symlink;
@ -2895,7 +2884,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
uint32_t mapping_max;
blockmax = 0;
paritymax = 0;
count_file = 0;
count_hardlink = 0;
count_symlink = 0;
@ -3098,10 +3086,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
switch (c) {
case 'b' :
block_state_set(block, BLOCK_STATE_BLK);
/* keep track of the required parity size */
if (v_pos + 1 > paritymax)
paritymax = v_pos + 1;
break;
case 'n' :
/* deprecated NEW blocks are converted to CHG ones */
@ -3809,21 +3793,18 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
}
/* check that the stored parity size matches the loaded state */
if (blockmax != parity_size(state)) {
if (blockmax != parity_allocated_size(state)) {
/* LCOV_EXCL_START */
fprintf(stderr, "Internal inconsistency in parity size in '%s' at offset %" PRIi64 "\n", path, stell(f));
if (state->opt.skip_content_check) {
fprintf(stderr, "Overriding from %u to %u.\n", blockmax, parity_size(state));
blockmax = parity_size(state);
fprintf(stderr, "Overriding from %u to %u.\n", blockmax, parity_allocated_size(state));
blockmax = parity_allocated_size(state);
} else {
exit(EXIT_FAILURE);
}
/* LCOV_EXCL_STOP */
}
/* set the required parity size */
state->loaded_paritymax = paritymax;
if (state->opt.verbose) {
printf("%8u files\n", count_file);
printf("%8u hardlinks\n", count_hardlink);
@ -3853,7 +3834,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
count_dir = 0;
/* blocks of all array */
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
/* clear the info for unused blocks */
/* and get some other info */

14
cmdline/state.h
View file

@ -150,20 +150,6 @@ struct snapraid_state {
*/
uint64_t tick_last;
/**
* Required size of the parity file, computed from the loaded state.
*
* This size only counts BLK blocks, ignoring CHG, REL and DELETED ones,
* because in such case the parity may be still not updated to contain them.
*
* In normal case it's also the blockmax size returned by parity_size().
* In case of interrupted sync, this is the position + 1 of the last BLK block.
* Potentionally smaller than parity_size().
*
* This value is used to check the minimal parity size before starting a sync.
*/
block_off_t loaded_paritymax;
int clear_past_hash; /**< Clear all the hash from CHG and DELETED blocks when reading the state from an incomplete sync. */
time_t progress_whole_start; /**< Initial start of the whole process. */

2
cmdline/status.c
View file

@ -80,7 +80,7 @@ int state_status(struct snapraid_state* state)
/* keep track if at least a free info is available */
free_not_zero = 0;
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
fprintf(stdlog, "summary:block_size:%u\n", state->block_size);
fprintf(stdlog, "summary:parity_block_count:%u\n", blockmax);

51
cmdline/sync.c
View file

@ -1181,7 +1181,8 @@ bail:
int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount)
{
block_off_t blockmax;
data_off_t loaded_size;
block_off_t used_paritymax;
block_off_t file_paritymax;
data_off_t size;
data_off_t out_size;
int ret;
@ -1195,9 +1196,14 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
printf("Initializing...\n");
blockmax = parity_size(state);
blockmax = parity_allocated_size(state);
size = blockmax * (data_off_t)state->block_size;
loaded_size = state->loaded_paritymax * (data_off_t)state->block_size;
/* minum size of the parity files we expect */
used_paritymax = parity_used_size(state);
/* effective size of the parity files */
file_paritymax = 0;
if (blockstart > blockmax) {
/* LCOV_EXCL_START */
@ -1212,6 +1218,8 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
for (l = 0; l < state->level; ++l) {
block_off_t parityblocks;
/* create the file and open for writing */
parity_ptr[l] = &parity[l];
ret = parity_create(parity_ptr[l], state->parity[l].path, &out_size, state->file_mode);
@ -1222,20 +1230,41 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
/* LCOV_EXCL_STOP */
}
/* if we do a full sync, having a wrong parity size is expected */
if (!state->opt.force_full) {
/* if the file is too small */
if (out_size < loaded_size) {
/* LCOV_EXCL_START */
fprintf(stderr, "DANGER! The %s file %s is smaller than the expected %" PRId64 ".\n", lev_name(l), state->parity[l].path, loaded_size);
/* number of block in the parity file */
parityblocks = out_size / state->block_size;
/* if the file is too small */
if (parityblocks < used_paritymax) {
fprintf(stderr, "WARNING! The %s file %s has data only %u blocks instead of %u.\n", lev_name(l), state->parity[l].path, parityblocks, used_paritymax);
}
/* keep the smallest parity number of blocks */
if (l == 0 || file_paritymax > parityblocks)
file_paritymax = parityblocks;
}
/* if we do a full sync, having a wrong parity size is expected */
if (!state->opt.force_full) {
/* if the parities are too small */
if (file_paritymax < used_paritymax) {
/* LCOV_EXCL_START */
fprintf(stderr, "DANGER! One or more the parity files are smaller than expected!\n");
if (file_paritymax != 0) {
fprintf(stderr, "If this happens because you are using an old content file,\n");
fprintf(stderr, "you can 'sync' anyway using 'snapraid --force-full sync'\n");
fprintf(stderr, "to force a full rebuild of the parity.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
} else {
fprintf(stderr, "It's possible that the parity disks are not mounted.\n");
fprintf(stderr, "If instead you are adding a new parity level, you can 'sync' using\n");
fprintf(stderr, "'snapraid --force-full sync' to force a full rebuild of the parity.\n");
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* now change the size of all parities */
for (l = 0; l < state->level; ++l) {
/* change the size of the parity file, truncating or extending it */
/* from this point all the DELETED blocks after the end of the parity are invalid */
/* and they are automatically removed when we save the new content file */