diff --git a/Makefile.am b/Makefile.am index 09207729..ce0ff94e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -82,6 +82,17 @@ check-local: $(TESTENV) ./mktest$(EXEEXT) generate 3 3 100 4000 $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf sync $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check + echo --- Delete some files and create some new, sync and check in multiple steps + rm test/disk1/4* + rm test/disk2/5* + rm test/disk2/6* + $(TESTENV) ./mktest$(EXEEXT) generate 4 3 100 4000 + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf -t 7 sync + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf -t 11 sync + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf sync + $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Delete some files, fix and check rm test/disk1/1* $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null diff --git a/check.c b/check.c index 6f391d22..5163dcfe 100644 --- a/check.c +++ b/check.c @@ -66,6 +66,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity struct snapraid_block* failed_block; struct snapraid_handle* failed_handle; int one_hash; + int all_parity; /* for each disk */ one_hash = 0; @@ -73,6 +74,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity struct snapraid_block* block = disk_block_get(handle[j].disk, i); if (block && bit_has(block->flag, BLOCK_HAS_HASH)) { one_hash = 1; + break; } } @@ -83,10 +85,12 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity /* start with 0 */ memset(xor_buffer, 0, state->block_size); - /* for each disk, process the block */ - failed = 0; - failed_block = 0; + all_parity = 1; /* if all hashed block have parity computed */ + failed = 0; /* number of failed block */ + failed_block = 0; /* last failed block */ failed_handle = 0; + + /* for each disk, process the block */ for(j=0;jflag, BLOCK_HAS_HASH)) continue; + /* keep track if at least one hashed block has no parity */ + if (!bit_has(block->flag, BLOCK_HAS_PARITY)) + all_parity = 0; + ret = handle_close_if_different(&handle[j], block->file); if (ret == -1) { fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); @@ -170,7 +178,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity count_size += read_size; } - if (parity_f == -1) { + if (parity_f == -1 || !all_parity) { /* all the cases with no parity file */ if (failed != 0) { fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i); @@ -320,7 +328,7 @@ bail: return 0; } -void state_check(struct snapraid_state* state, int fix, block_off_t blockstart) +void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount) { char path[PATH_MAX]; block_off_t blockmax; @@ -342,6 +350,11 @@ void state_check(struct snapraid_state* state, int fix, block_off_t blockstart) exit(EXIT_FAILURE); } + /* adjust the number of block to process */ + if (blockcount != 0 && blockstart + blockcount < blockmax) { + blockmax = blockstart + blockcount; + } + pathcpy(path, sizeof(path), state->parity); if (fix) { /* if fixing, create the file and open for writing */ diff --git a/snapraid.1 b/snapraid.1 index 3a618578..dc7900c0 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -2,13 +2,17 @@ .SH NAME snapraid \(hy SnapRAID Backup For Disk Arrays .SH SYNOPSIS -snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hys, \(hy\(hystart BLOCK] +snapraid [\(hyc, \(hy\(hyconf CONFIG] .PD 0 .PP .PD [\(hyZ, \(hy\(hyforce\(hyzero] [\(hyE, \(hy\(hyforce\(hyempty] .PD 0 .PP +.PD + [\(hys, \(hy\(hystart BLKSTART] [\(hyt, \(hy\(hycount BLKCOUNT] +.PD 0 +.PP .PD [\(hyv, \(hy\(hyverbose] .PD 0 @@ -237,11 +241,6 @@ Selects the configuration file. If not specified is assumed the file \(cq/etc/snapraid.conf\(cq in Unix, and \(cqsnapraid.conf\(cq in Windows. .TP -.B \(hys, \(hy\(hystart BLOCK -Starts the processing from the specified -block number. It could be useful to easy retry to check or -fix some specific block, in case of a damaged disk. -.TP .B \(hyZ, \(hy\(hyforce\(hyzero Forces the insecure operation of syncing a file with zero size that before was not empty. @@ -258,6 +257,15 @@ unless you specify this option. This allows to easily detect when a data file\(hysystem is not mounted. .TP +.B \(hys, \(hy\(hystart BLKSTART +Starts the processing from the specified +block number. It could be useful to easy retry to check +or fix some specific block, in case of a damaged disk. +.TP +.B \(hyt, \(hy\(hycount BLKCOUNT +Process only the specified number of blocks. +It\(cqs present mainly for testing. +.TP .B \(hyv, \(hy\(hyverbose Prints more information in the processing. .TP diff --git a/snapraid.c b/snapraid.c index 1ff86f4c..1a23333b 100644 --- a/snapraid.c +++ b/snapraid.c @@ -43,19 +43,21 @@ void usage(void) { printf("Usage: " PACKAGE " [options]\n"); printf("\n"); printf("Options:\n"); - printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file (default " CONF ")\n"); - printf(" " SWITCH_GETOPT_LONG("-s, --start BLK ", "-s") " Start from the specified block number\n"); - printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n"); - printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force synching of disks that get empty\n"); - printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); - printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n"); - printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n"); + printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file (default " CONF ")\n"); + printf(" " SWITCH_GETOPT_LONG("-s, --start BLKSTART", "-s") " Start from the specified block number\n"); + printf(" " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT", "-t") " Count of block to process\n"); + printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n"); + printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force synching of disks that get empty\n"); + printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); + printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n"); + printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n"); } #if HAVE_GETOPT_LONG struct option long_options[] = { { "conf", 1, 0, 'c' }, { "start", 1, 0, 's' }, + { "count", 1, 0, 't' }, { "force-zero", 0, 0, 'Z' }, { "force-empty", 0, 0, 'E' }, { "verbose", 0, 0, 'v' }, @@ -65,7 +67,7 @@ struct option long_options[] = { }; #endif -#define OPTIONS "c:s:ZEvhV" +#define OPTIONS "c:s:t:ZEvhV" volatile int global_interrupt = 0; @@ -92,6 +94,7 @@ int main(int argc, char* argv[]) struct snapraid_state state; int operation; block_off_t blockstart; + block_off_t blockcount; int ret; /* intercept Ctrl+C */ @@ -103,6 +106,7 @@ int main(int argc, char* argv[]) force_zero = 0; force_empty = 0; blockstart = 0; + blockcount = 0; opterr = 0; while ((c = @@ -122,6 +126,12 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } break; + case 't' : + if (stru32(optarg, &blockcount) != 0) { + fprintf(stderr, "Invalid count number '%s'\n", optarg); + exit(EXIT_FAILURE); + } + break; case 'Z' : force_zero = 1; break; @@ -168,7 +178,7 @@ int main(int argc, char* argv[]) state_scan(&state); - ret = state_sync(&state, blockstart); + ret = state_sync(&state, blockstart, blockcount); /* save the new state if required */ if (state.need_write) @@ -180,7 +190,7 @@ int main(int argc, char* argv[]) } else { state_read(&state); - state_check(&state, operation == OPERATION_FIX, blockstart); + state_check(&state, operation == OPERATION_FIX, blockstart, blockcount); } state_done(&state); diff --git a/snapraid.d b/snapraid.d index 4053be62..6703e8e0 100644 --- a/snapraid.d +++ b/snapraid.d @@ -2,8 +2,9 @@ Name{number} snapraid - SnapRAID Backup For Disk Arrays Synopsis - :snapraid [-c, --conf CONFIG] [-s, --start BLOCK] + :snapraid [-c, --conf CONFIG] : [-Z, --force-zero] [-E, --force-empty] + : [-s, --start BLKSTART] [-t, --count BLKCOUNT] : [-v, --verbose] : COMMAND @@ -155,11 +156,6 @@ Options the file '/etc/snapraid.conf' in Unix, and 'snapraid.conf' in Windows. - -s, --start BLOCK - Starts the processing from the specified - block number. It could be useful to easy retry to check or - fix some specific block, in case of a damaged disk. - -Z, --force-zero Forces the insecure operation of syncing a file with zero size that before was not empty. @@ -176,6 +172,15 @@ Options This allows to easily detect when a data file-system is not mounted. + -s, --start BLKSTART + Starts the processing from the specified + block number. It could be useful to easy retry to check + or fix some specific block, in case of a damaged disk. + + -t, --count BLKCOUNT + Process only the specified number of blocks. + It's present mainly for testing. + -v, --verbose Prints more information in the processing. diff --git a/snapraid.html b/snapraid.html deleted file mode 100644 index b0e4443a..00000000 --- a/snapraid.html +++ /dev/null @@ -1,521 +0,0 @@ - - -SnapRAID Backup For Disk Arrays - - -

SnapRAID Backup For Disk Arrays

-

1 -Synopsis -

-
-snapraid [-c, --conf CONFIG] [-s, --start BLOCK]
-    [-Z, --force-zero] [-E, --force-empty]
-    [-v, --verbose]
-    COMMAND
-
-

-

-snapraid [-V, --version] [-h, --help]
-
-

2 -Description -

-SnapRAID is a backup program for a disk array using redundancy. -

-SnapRAID uses a disk of the array to store redundancy information, -and it allows to recover from a disk failure. -

-SnapRAID is mainly targeted for a home media center, where you have -a lot of big files that rarely change. -

-The main features of SnapRAID are: -

-

-The official site of SnapRAID is: -

-
-
-http://snapraid.sourceforge.net
-
-
-

3 -Limitations -

-SnapRAID is in between a RAID and a backup program trying to get the best -benefits of them. Altought it also has some limitations that you should -consider before using it. -

-The main one, is that if a disk fail, and you haven't recently synched, -you may not able to do a complete recover. -More specifically, you may be unable to recover up to the size of the -amount of the changed or deleted files, from the last sync operation. -This happens even if the files changed or deleted are not in the -failed disk. -New added files don't prevent the recovering of the already existing -files. You may only lose the just added files, if they were on the failed -disk. -

-This is the reason because SnapRAID is better suited for data that -rarely change. -

-Other limitations are: -* You have different filesystems for each disk. -
-Using a RAID you have only a big filesystem. -
-

-

4 -Getting Started -

-To use SnapRAID you need first select one disk of your disk array -to dedicate at the redundancy information. -

-You have to pick the biggest disk in the array, as the redundancy -information may grow in size as the biggest data disk in the array. -

-This disk will be dedicated to this pourpuse only and you should -not store other data on it. -

-Suppose now that you have mounted all your disks in the mount points: -

-
-
-/mnt/diskpar <- selected disk for parity
-/mnt/disk1
-/mnt/disk2
-/mnt/disk3
-
-
-

-you have to create the configuration file /etc/snapraid.conf with -the following content: -

-
-
-parity /mnt/diskpar/parity
-content /mnt/diskpar/content
-disk d1 /mnt/disk1/
-disk d2 /mnt/disk2/
-disk d3 /mnt/disk3/
-
-
-

-At this point you are ready to start the "sync" command to build the -redundancy information. -

-
-
-snapraid sync
-
-
-

-This process maty take some hours the first time, depending on the size -of the data already present in the disks. If the disks are empty -the process is immediate. -

-You can stop it at any time pressing Ctrl+C, and at the next run it -will start where interrupted. -

-When this command completes, your data is SAFE. -

-At this point you can start using your data as you like, and peridiocally -update the redundancy information running the "sync" command. -

-To check the integrity of your data you can use the "check" command: -

-
-
-snapraid check
-
-
-

-If will read all your data, to check if it's correct. -

-If an error is found, you can use the "fix" command to fix it. -

-
-
-snapraid fix
-
-
-

-Note that the fix command will revert your data at the state of the -last "sync" command executed. It works like a snapshot was taken -in "sync". -

-In this regard snapraid is more like a backup program than a RAID -system. For example, you can use it to recover from an accidentally -deleted directory, simply running the fix command. -

5 -Commands -

-SnapRAID provides three simple commands that allow to: - - - -
-sync -
-Updates the redundancy information. All the modified files -in the disk array are read, and the redundancy data is -updated. -You can stop this process at any time pressing Ctrl+C, -without losing the work already done. -
- - -
-check -
-Checks all the files the redundancy data. All the files -are hashed and compared with the snapshot saved in the -previous "sync" command. -
- - -
-fix -
-Checks and fix all the files. It's like "check" but it -also tries to fix problems reverting the state of the -disk array at the previous "sync" command. -
-

6 -Options -

- - -
--c, --conf CONFIG -
-Selects the configuration file. If not specified is assumed -the file `/etc/snapraid.conf'. -
- - -
--s, --start BLOCK -
-Starts the processing from the specified -block number. It could be useful to easy retry to check or -fix some specific block, in case of a damaged disk. -
- - -
--Z, --force-zero -
-Forces the insecure operation of synching a file with zero -size that before was not empty. -If SnapRAID detects such condition, it stops proceeding -unless you specify this option. -This allows to easily detect when after a system crash, -some accessed files were zeroed. -
- - -
--E, --force-empty -
-Forces the insecure operation of synching an empty disk -that before was not empty. -If SnapRAID detects such condition, it stops proceeding -unless you specify this option. -This allows to easily detect when a data filesystem is not -mounted. -
- - -
--v, --verbose -
-Prints more information in the processing. -
- - -
--h, --help -
-Prints a short help screen. -
- - -
--V, --version -
-Prints the program version. -
-

7 -Configuration -

-SnapRAID requires a configuration file to know where your disk array -is located, and where storing the redundancy information. -

-This configuration file is located in /etc/snapraid.conf and -it should contains the following options: - - -
-parity FILE -
-Defines the file to use to store the redundancy information. -It must be placed in a disk dedicated for this porpose with -as much free space as the biggest disk in the array. -Leaving the parity disk reserved for only this file, ensures that -it doesn't get fragmented, improving the performance. -This option can be used only one time. -
- - -
-content FILE -
-Defines the file to use to store the content of the redundancy -organization. -It can be placed in the same disk of the parity file, or better -in another disk, but NOT in a data disk of the array. -This option can be used only one time. -
- - -
-disk NAME DIR -
-Defines the name and the mount point of the disks of the array. -NAME is used to identify the disk, and it must be unique. -DIR is the mount point of the disk in the filesystem. -You can change the mount point as you like, as far you -keep the NAME fixed. -You should use one option for each disk of the array. -
- - -
-exclude PATTERN -
-Defines the file or directory patterns to exclude from the sync -process. See the PATTERN section for more details in the -pattern specifications. -This option can be used many times. -
- - -
-block_size SIZE_IN_KIBIBYTES -
-Defines the basic block size in kibi bytes of -the redundancy blocks. Where one kibi bytes is 1024 bytes. -The default is 256 and it should work for most conditions. -You increase this value if you do not have enough memory -to run SnapRAID. -It requires to run something about TS*24/BS bytes, where TS -is the total size in bytes of your disk array, and BS is the -block size in bytes. -
-

-
-For example with 6 disk of 2 TiB and a block size of 256 KiB -(1 KiB = 1024 Bytes) you have: -
-

-
-
-memory = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB
-
-
-

-
-You should instead decrease this value if you have a lot of -small files in the disk array. For each file, even if of few -bytes, a whole block is always allocated, so you may have a lot -of unused space. -
-

-An example of a typical configuration is: -

-
-
-parity /mnt/diskpar/parity
-content /mnt/diskpar/content
-disk d1 /mnt/disk1/
-disk d2 /mnt/disk2/
-disk d3 /mnt/disk3/
-exclude *.bak
-exclude /lost+found/
-exclude tmp/
-block_size 256
-
-
-

8 -Pattern -

-Patterns are used to define the files and directories to exclude -from the redundancy process. -

-It makes sense to exclude any file not worth to be saved or that -changes often. -

-There are four different types of patterns: - - -
-FILE -
-Excludes any file named as FILE. You can use any globbing -character like * and ?. -This pattern is applied only to files and not to directories. -
- - -
-DIR/ -
-Excludes any directory named DIR. You can use any globbing -character like * and ?. -This pattern is applied only to directories and not to files. -
- - -
-/PATH/FILE -
-Excludes the exact specified file path. You can use any -globbing character like * and ? but they never matches a -directory slash. -This pattern is applied only to files and not to directories. -
- - -
-/PATH/DIR/ -
-Excludes the exact specified directory path. You can use any -globbing character like * and ? but they never matches a -directory slash. -This pattern is applied only to directories and not to files. -
-

-For example: -
-
-# Excludes any file named "*.bak"
-exclude *.bak
-# Excludes the root directory "/lost+found"
-exclude /lost+found/
-# Excludes any directory named "tmp"
-exclude tmp/
-
-
-

9 -Content -

-SnapRAID creates a content file describing the content of your disk array. -

-You should never change it manually, altought the format of this file is -described here. - - -
-blk_size SIZE -
-Defines the size of the block in bytes. It must match the size -defined in the configuration file. -
- - -
-file DISK SIZE TIME INODE PATH -
-Defines a file in the specified DISK. -The INODE number is used to identify the file in the "sync" -command, allowing to rename or move the file in disk without -the need to recompute the parity for it. -The SIZE and TIME information are used to identify if the file -changed from the last "sync" command, and if there is the need -to recompute the parity. -The PATH information is used in the "check" and "fix" commands -to identify the file. -
- - -
-blk BLOCK HASH -
-Defines the ordered parity block list used by the last defined file. -BLOCK is the block position in the "parity" file. -0 for the first block, 1 for the second one and so on. -HASH is the md5 of the block. In the last block of the file, -the HASH is the hash of only the used part of the block. -
-

10 -Parity -

-SnapRAID creates a parity file containing the parity information of your disk array. -

-It's a binary file, containing the parity information of all the blocks defined -in the "content" file. -

-For all the blocks at a given position, the parity information is computed with -the XOR operator applied to all the blocks. -

-When a file block is shorter than the default block size, for example because it's -the last block of a file, it's assumed filled with 0 at the end in the XOR operation. -

11 -Copyright -

-This file is Copyright (C) 2011 Andrea Mazzoleni -

12 -See Also -

-rsync(1) - - diff --git a/snapraid.txt b/snapraid.txt index eca48909..9f01943c 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -6,8 +6,9 @@ 1 SYNOPSIS ========== -snapraid [-c, --conf CONFIG] [-s, --start BLOCK] +snapraid [-c, --conf CONFIG] [-Z, --force-zero] [-E, --force-empty] + [-s, --start BLKSTART] [-t, --count BLKCOUNT] [-v, --verbose] COMMAND @@ -174,11 +175,6 @@ SnapRAID provides three simple commands that allow to: the file '/etc/snapraid.conf' in Unix, and 'snapraid.conf' in Windows. - -s, --start BLOCK - Starts the processing from the specified - block number. It could be useful to easy retry to check or - fix some specific block, in case of a damaged disk. - -Z, --force-zero Forces the insecure operation of syncing a file with zero size that before was not empty. @@ -195,6 +191,15 @@ SnapRAID provides three simple commands that allow to: This allows to easily detect when a data file-system is not mounted. + -s, --start BLKSTART + Starts the processing from the specified + block number. It could be useful to easy retry to check + or fix some specific block, in case of a damaged disk. + + -t, --count BLKCOUNT + Process only the specified number of blocks. + It's present mainly for testing. + -v, --verbose Prints more information in the processing. diff --git a/state.h b/state.h index b79ba1a5..89fa4cdb 100644 --- a/state.h +++ b/state.h @@ -71,12 +71,12 @@ void state_scan(struct snapraid_state* state); /** * Syncs the parity data. */ -int state_sync(struct snapraid_state* state, block_off_t blockstart); +int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount); /** * Checks (and fixes) all the files and the parity data. */ -void state_check(struct snapraid_state* state, int fix, block_off_t blockstart); +void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount); /** * Writes the progress. diff --git a/sync.c b/sync.c index c7bbf9c5..2d7e8d51 100644 --- a/sync.c +++ b/sync.c @@ -217,7 +217,7 @@ bail: return 0; } -int state_sync(struct snapraid_state* state, block_off_t blockstart) +int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount) { char path[PATH_MAX]; block_off_t blockmax; @@ -236,6 +236,11 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart) fprintf(stderr, "Error in the starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax); exit(EXIT_FAILURE); } + + /* adjust the number of block to process */ + if (blockcount != 0 && blockstart + blockcount < blockmax) { + blockmax = blockstart + blockcount; + } pathcpy(path, sizeof(path), state->parity); f = parity_create(path, size);