rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds the new autosave configuration option.

Browse source 
Checks the parity validity invariant after reading the content file.
This commit is contained in:
Andrea Mazzoleni 2012-03-13 23:59:59 +01:00
parent e927c8d905
commit 2a50ec747d
11 changed files with 209 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

6
elem.h
View file

@ -80,6 +80,9 @@ struct snapraid_file;
/**
* The block has the hash computed, but the parity is invalid.
* This happens when the parity is invalidated by a change in another block.
* Note that this state is almost reduntant because we can detect that the parity is invalid
* also checking the other disks, where for sure exists a block the BLOCK_STATE_CHG/NEW state
* or a BLOCK_DELETED block.
*/
#define BLOCK_STATE_INV 2
@ -300,6 +303,9 @@ block_off_t block_file_pos(struct snapraid_block* block);
*/
int block_is_last(struct snapraid_block* block);
/**
* Checks if the block is a real allocated block.
*/
static inline int block_is_valid(struct snapraid_block* block)
{
return block != BLOCK_EMPTY && block != BLOCK_DELETED;

4
parity.c
View file

@ -37,7 +37,9 @@ block_off_t parity_resize(struct snapraid_state* state)
/* start from the declared size */
block_off_t block = tommy_array_size(&disk->blockarr);
/* decrease the block until an allocated or deleted block */
/* decrease the block until an allocated one */
/* we don't stop at deleted blocks, because we want to have them cleared */
/* if they are at the end of the parity */
while (block > parity_block && !block_is_valid(tommy_array_get(&disk->blockarr, block - 1)))
--block;

26
snapraid.1
View file

@ -353,7 +353,7 @@ This option can be used many times.
In Unix, ensure to quote globbing chars if used.
.TP
.B \(hyH, \(hy\(hyfilter\(hynohidden
Filter out hidden files and directory. In Unix hidden files are
Filters out hidden files and directory. In Unix hidden files are
the ones starting with \(cq.\(cq. In Windows they are the ones with the
hidden attribute.
.TP
@ -482,8 +482,8 @@ The default is 256 and it should work for most conditions.
You could increase this value if you do not have enough RAM
memory to run SnapRAID.
.PP
As a rule of thumb, with 4GB or more memory use the default 256,
with 2GB use 512, and with 1GB use 1024.
As a rule of thumb, with 4 GiB or more memory use the default 256,
with 2 GiB use 512, and with 1 GiB use 1024.
.PP
In more details SnapRAID requires about TS*24/BS bytes
of RAM memory to run. Where TS is the total size in bytes of
@ -506,6 +506,13 @@ wasted for each file.
.PP
For example, with 10000 files and a 256 KiB block size, you are
going to waste 1.2 GiB.
.SS autosave SIZE_IN_GIBIBYTES
Automatically save the state when synching after the specied amount
of GiB processed.
This option is useful to avoid to restart from scratch long \(cqsync\(cq
commands interrupted by a machine crash.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
.SS Examples
An example of a typical configuration for Unix is:
.PP
@ -765,7 +772,7 @@ snapraid fix 2>fix.log
.PP
This command will take a long time.
.PP
Take care that you need also few GB free to store the fix.log file, so run it
Take care that you need also few gigabytes free to store the fix.log file, so run it
from a disk with some free space.
.PP
Now you have recovered all the recoverable. If some file is partially or totally
@ -844,7 +851,7 @@ to recompute the parity.
The PATH information is used in the \(a"check\(a" and \(a"fix\(a" commands
to identify the file.
.SS blk BLOCK HASH
Defines the ordered parity block list used by the last defined file.
Defines an ordered parity block, part of the last defined file.
.PP
BLOCK is the block position in the \(a"parity\(a" file.
0 for the first block, 1 for the second one and so on.
@ -858,13 +865,15 @@ This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS new BLOCK
Like \(a"blk\(a", but for new allocated blocks for which the hash is not
yet computed.
yet computed, and the stored parity doesn\(cqt take into account this
new block.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS chg BLOCK
Like \(a"blk\(a", but for reallocated blocks for which the hash is not
yet computed.
yet computed, and the parity is computed using the previous value
of the block.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.
@ -874,7 +883,8 @@ Defines the list of blocks that are deleted from a disk.
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS off BLOCK
Defines the block deleted from a disk.
Defines a block deleted from a disk, part of the last defined hole,
for which the parity is computed using the previous value.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.

4
snapraid.c
View file

@ -124,8 +124,8 @@ int main(int argc, char* argv[])
int gui;
int force_zero;
int force_empty;
int find_by_name;
int filter_hidden;
int find_by_name;
int test_expect_unrecoverable;
int test_expect_recoverable;
int test_kill_after_sync;
@ -146,8 +146,8 @@ int main(int argc, char* argv[])
gui = 0;
force_zero = 0;
force_empty = 0;
find_by_name = 0;
filter_hidden = 0;
find_by_name = 0;
test_expect_unrecoverable = 0;
test_expect_recoverable = 0;
test_kill_after_sync = 0;

9
snapraid.conf.example
View file

@ -47,3 +47,12 @@ exclude /lost+found/
# Format: "block_size SIZE_IN_KiB"
block_size 256
# Automatically save the state when synching after the specied amount
# of GiB processed.
# This option is useful to avoid to restart from scratch long 'sync'
# commands interrupted by a machine crash.
# The SIZE argument is specified in gibi bytes -> 1073741824 bytes
# Default value is 0, meaning disabled.
# Format: "autosave_size SIZE_IN_GiB"
# autosize 500

9
snapraid.conf.example.windows
View file

@ -48,3 +48,12 @@ exclude \System Volume Information
# Format: "block_size SIZE_IN_KiB"
block_size 256
# Automatically save the state when synching after the specied amount
# of GiB processed.
# This option is useful to avoid to restart from scratch long 'sync'
# commands interrupted by a machine crash.
# The SIZE argument is specified in gibi bytes -> 1073741824 bytes
# Default value is 0, meaning disabled.
# Format: "autosave_size SIZE_IN_GiB"
# autosize 500

27
snapraid.d
View file

@ -238,7 +238,7 @@ Options
In Unix, ensure to quote globbing chars if used.
-H, --filter-nohidden
Filter out hidden files and directory. In Unix hidden files are
Filters out hidden files and directory. In Unix hidden files are
the ones starting with '.'. In Windows they are the ones with the
hidden attribute.
@ -375,8 +375,8 @@ Configuration
You could increase this value if you do not have enough RAM
memory to run SnapRAID.
As a rule of thumb, with 4GB or more memory use the default 256,
with 2GB use 512, and with 1GB use 1024.
As a rule of thumb, with 4 GiB or more memory use the default 256,
with 2 GiB use 512, and with 1 GiB use 1024.
In more details SnapRAID requires about TS*24/BS bytes
of RAM memory to run. Where TS is the total size in bytes of
@ -397,6 +397,14 @@ Configuration
For example, with 10000 files and a 256 KiB block size, you are
going to waste 1.2 GiB.
autosave SIZE_IN_GIBIBYTES
Automatically save the state when synching after the specied amount
of GiB processed.
This option is useful to avoid to restart from scratch long 'sync'
commands interrupted by a machine crash.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
Examples
An example of a typical configuration for Unix is:
@ -523,7 +531,7 @@ Recovering
This command will take a long time.
Take care that you need also few GB free to store the fix.log file, so run it
Take care that you need also few gigabytes free to store the fix.log file, so run it
from a disk with some free space.
Now you have recovered all the recoverable. If some file is partially or totally
@ -601,7 +609,7 @@ Content
to identify the file.
blk BLOCK HASH
Defines the ordered parity block list used by the last defined file.
Defines an ordered parity block, part of the last defined file.
BLOCK is the block position in the "parity" file.
0 for the first block, 1 for the second one and so on.
@ -617,14 +625,16 @@ Content
new BLOCK
Like "blk", but for new allocated blocks for which the hash is not
yet computed.
yet computed, and the stored parity doesn't take into account this
new block.
This field is used only when you interrupt manually the "sync"
command.
chg BLOCK
Like "blk", but for reallocated blocks for which the hash is not
yet computed.
yet computed, and the parity is computed using the previous value
of the block.
This field is used only when you interrupt manually the "sync"
command.
@ -636,7 +646,8 @@ Content
command.
off BLOCK
Defines the block deleted from a disk.
Defines a block deleted from a disk, part of the last defined hole,
for which the parity is computed using the previous value.
This field is used only when you interrupt manually the "sync"
command.

27
snapraid.txt
View file

@ -267,7 +267,7 @@ SnapRAID provides the following options:
In Unix, ensure to quote globbing chars if used.
-H, --filter-nohidden
Filter out hidden files and directory. In Unix hidden files are
Filters out hidden files and directory. In Unix hidden files are
the ones starting with '.'. In Windows they are the ones with the
hidden attribute.
@ -421,8 +421,8 @@ The default is 256 and it should work for most conditions.
You could increase this value if you do not have enough RAM
memory to run SnapRAID.
As a rule of thumb, with 4GB or more memory use the default 256,
with 2GB use 512, and with 1GB use 1024.
As a rule of thumb, with 4 GiB or more memory use the default 256,
with 2 GiB use 512, and with 1 GiB use 1024.
In more details SnapRAID requires about TS*24/BS bytes
of RAM memory to run. Where TS is the total size in bytes of
@ -443,6 +443,14 @@ wasted for each file.
For example, with 10000 files and a 256 KiB block size, you are
going to waste 1.2 GiB.
---- 7.7.1 autosave SIZE_IN_GIBIBYTES ----
Automatically save the state when synching after the specied amount
of GiB processed.
This option is useful to avoid to restart from scratch long 'sync'
commands interrupted by a machine crash.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
7.8 Examples
------------
@ -577,7 +585,7 @@ Run the fix command, storing the log in an external file with:
This command will take a long time.
Take care that you need also few GB free to store the fix.log file, so run it
Take care that you need also few gigabytes free to store the fix.log file, so run it
from a disk with some free space.
Now you have recovered all the recoverable. If some file is partially or totally
@ -670,7 +678,7 @@ to identify the file.
10.6 blk BLOCK HASH
-------------------
Defines the ordered parity block list used by the last defined file.
Defines an ordered parity block, part of the last defined file.
BLOCK is the block position in the "parity" file.
0 for the first block, 1 for the second one and so on.
@ -690,7 +698,8 @@ command.
--------------
Like "blk", but for new allocated blocks for which the hash is not
yet computed.
yet computed, and the stored parity doesn't take into account this
new block.
This field is used only when you interrupt manually the "sync"
command.
@ -699,7 +708,8 @@ command.
--------------
Like "blk", but for reallocated blocks for which the hash is not
yet computed.
yet computed, and the parity is computed using the previous value
of the block.
This field is used only when you interrupt manually the "sync"
command.
@ -715,7 +725,8 @@ command.
10.11 off BLOCK
---------------
Defines the block deleted from a disk.
Defines a block deleted from a disk, part of the last defined hole,
for which the parity is computed using the previous value.
This field is used only when you interrupt manually the "sync"
command.

93
state.c
View file

@ -29,6 +29,7 @@ void state_init(struct snapraid_state* state)
state->force_empty = 0;
state->filter_hidden = 0;
state->find_by_name = 0;
state->autosave = 0;
state->expect_unrecoverable = 0;
state->expect_recoverable = 0;
state->need_write = 0;
@ -387,6 +388,24 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
exit(EXIT_FAILURE);
}
tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
} else if (strcmp(tag, "autosave") == 0) {
char* e;
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'autosave' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
state->autosave = strtoul(buffer, &e, 0);
if (!e || *e) {
fprintf(stderr, "Invalid 'autosave' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
/* convert to GiB */
state->autosave *= 1024 * 1024 * 1024;
} else if (tag[0] == 0) {
/* allow empty lines */
} else if (tag[0] == '#') {
@ -525,6 +544,7 @@ static void state_map(struct snapraid_state* state)
static void state_content_check(struct snapraid_state* state, const char* path)
{
tommy_node* i;
block_off_t j;
/* checks that any map has different name and position */
for(i=state->maplist;i!=0;i=i->next) {
@ -542,6 +562,48 @@ static void state_content_check(struct snapraid_state* state, const char* path)
}
}
}
/* check the parity validity invariant */
for(j=0;1;++j) { /* for all the blocks */
int at_least_one = 0;
int parity_is_valid = 0;
int parity_is_invalid = 0;
/* for all the disks */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* disk = i->data;
/* if the block is present */
block_off_t size = tommy_array_size(&disk->blockarr);
if (j < size) {
struct snapraid_block* block = tommy_array_get(&disk->blockarr, j);
at_least_one = 1;
if (block == BLOCK_EMPTY) {
/* not relevant */
} else if (block == BLOCK_DELETED) {
parity_is_invalid = 1;
} else {
switch (block_state_get(block)) {
case BLOCK_STATE_BLK : parity_is_valid = 1; break;
case BLOCK_STATE_NEW : parity_is_invalid = 1; break;
case BLOCK_STATE_INV : parity_is_invalid = 1; break;
case BLOCK_STATE_CHG : parity_is_invalid = 1; break;
}
}
}
}
if (parity_is_valid && parity_is_invalid) {
fprintf(stderr, "Internal inconsistency in parity validity at block %u\n", j);
exit(EXIT_FAILURE);
}
/* stop if reached the end of all the arrays */
if (!at_least_one)
break;
}
}
void state_read(struct snapraid_state* state)
@ -735,6 +797,13 @@ void state_read(struct snapraid_state* state)
/* insert the block in the block array */
tommy_array_grow(&disk->blockarr, v_pos + 1);
/* deleted block must be empty blocks */
if (tommy_array_get(&disk->blockarr, v_pos) != 0) {
fprintf(stderr, "Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
tommy_array_set(&disk->blockarr, v_pos, BLOCK_DELETED);
} else if (strcmp(tag, "file") == 0) {
/* file */
@ -1442,6 +1511,7 @@ void state_progress_begin(struct snapraid_state* state, block_off_t blockstart,
state->progress_start = now;
state->progress_last = now;
state->progress_subtract = 0;
}
}
@ -1462,6 +1532,27 @@ void state_progress_end(struct snapraid_state* state, block_off_t countpos, bloc
#define PROGRESS_CLEAR " "
void state_progress_stop(struct snapraid_state* state)
{
time_t now;
now = time(0);
printf("\n");
state->progress_interruption = now;
}
void state_progress_restart(struct snapraid_state* state)
{
time_t now;
now = time(0);
if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
state->progress_subtract += now - state->progress_interruption;
}
int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
{
if (state->gui) {
@ -1473,7 +1564,7 @@ int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off
now = time(0);
if (state->progress_last != now) {
time_t delta = now - state->progress_start;
time_t delta = now - state->progress_start - state->progress_subtract;
printf("%u%%, %u MiB", countpos * 100 / countmax, (unsigned)(countsize / (1024*1024)));

13
state.h
View file

@ -36,6 +36,7 @@ struct snapraid_state {
int force_empty; /**< Forced dangerous operations of synching disk now empty. */
int filter_hidden; /**< Filter out hidden files. */
int find_by_name; /**< Forced dangerous operations of synching a rewritten disk. */
uint64_t autosave; /**< Autosave after the specified amount of data. 0 to disable. */
int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */
int expect_recoverable; /**< Expect presence of recoverable error in checking. */
int need_write; /**< If the state is changed. */
@ -52,6 +53,8 @@ struct snapraid_state {
tommy_list filterlist; /**< List of inclusion/exclusion. */
time_t progress_start; /**< Start of processing for progress visualization. */
time_t progress_last; /**< Last update of progress visualization. */
time_t progress_interruption; /**< Start of the measure interruption. */
time_t progress_subtract; /**< Time to subtract for the interruptions. */
};
/**
@ -124,5 +127,15 @@ void state_progress_end(struct snapraid_state* state, block_off_t countpos, bloc
*/
int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize);
/**
* Stop temporarely the progress.
*/
void state_progress_stop(struct snapraid_state* state);
/**
* Restart the progress.
*/
void state_progress_restart(struct snapraid_state* state);
#endif

19
sync.c
View file

@ -40,6 +40,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
uint64_t autosavesize;
int ret;
unsigned unrecoverable_error;
@ -84,6 +85,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
}
countsize = 0;
autosavesize = 0;
countpos = 0;
state_progress_begin(state, blockstart, blockmax, countmax);
for(i=blockstart;i<blockmax;++i) {
@ -128,6 +130,11 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* by default process the block, and skip it if something go wrong */
skip_this_block = 0;
/* compute the autosave size for all disk, even if not read */
/* this makes sense because the speed should be almost the same */
/* if the disks are read in parallel */
autosavesize += diskmax * state->block_size;
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
@ -295,6 +302,18 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (state_progress(state, i, countpos, countmax, countsize)) {
break;
}
/* autosave */
if (state->autosave != 0 && autosavesize >= state->autosave) {
autosavesize = 0;
state_progress_stop(state);
printf("Autosaving...\n");
state_write(state);
state_progress_restart(state);
}
}
state_progress_end(state, countpos, countmax, countsize);