From 0451c9e8a0a92955befd70c7a6d047c51ebf1003 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 22 Mar 2011 23:23:45 +0100 Subject: [PATCH] Handles correctly file deletion in sync. --- check.c | 22 ++++++------ elem.c | 2 +- elem.h | 5 ++- scan.c | 41 ++++++++++++++++++---- snapraid.html | 94 +++++++++++++++++++++++++++++++++------------------ state.c | 30 ++++++++++++---- sync.c | 17 +++++----- 7 files changed, 146 insertions(+), 65 deletions(-) diff --git a/check.c b/check.c index 2482f839..6f391d22 100644 --- a/check.c +++ b/check.c @@ -62,23 +62,22 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity last = start; for(i=blockstart;iis_hashed) { - hashed = 1; - break; + if (block && bit_has(block->flag, BLOCK_HAS_HASH)) { + one_hash = 1; } } - /* if there is at least an hashed block we have to check */ - if (!hashed) + /* if no hashed block skip */ + if (!one_hash) continue; /* start with 0 */ @@ -98,8 +97,10 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity if (!block) continue; - /* we can check and fix only if the block is hashed */ - if (!block->is_hashed) + /* we try to check and fix only if the block is hashed */ + /* it could be that a file was added, and not yet synched */ + /* so we should not include not hashed block in the parity computation */ + if (!bit_has(block->flag, BLOCK_HAS_HASH)) continue; ret = handle_close_if_different(&handle[j], block->file); @@ -385,3 +386,4 @@ void state_check(struct snapraid_state* state, int fix, block_off_t blockstart) if (unrecoverable_error != 0) exit(EXIT_FAILURE); } + diff --git a/elem.c b/elem.c index 92bab33a..976b92fe 100644 --- a/elem.c +++ b/elem.c @@ -145,7 +145,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t /* set the back pointer */ for(i=0;iblockmax;++i) { file->blockvec[i].parity_pos = POS_INVALID; - file->blockvec[i].is_hashed = 0; + file->blockvec[i].flag = 0; file->blockvec[i].file = file; } diff --git a/elem.h b/elem.h index 46355431..00a738c3 100644 --- a/elem.h +++ b/elem.h @@ -60,12 +60,15 @@ struct snapraid_filter { struct snapraid_file; +#define BLOCK_HAS_HASH 1 /**< If the hash value is valid. */ +#define BLOCK_HAS_PARITY 2 /**< If the parity block is valid. */ + /** * Block of a file. */ struct snapraid_block { block_off_t parity_pos; /**< Position of the block in the parity. */ - int is_hashed; /**< If the hash of the block is valid. */ + unsigned flag; /**< If the hash of the block is valid. */ struct snapraid_file* file; /**< Back pointer to the file owning this block. */ unsigned char hash[HASH_MAX]; /**< Hash of the block. */ }; diff --git a/scan.c b/scan.c index 2e3aabfe..a70575d9 100644 --- a/scan.c +++ b/scan.c @@ -33,7 +33,11 @@ struct snapraid_scan { */ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) { - block_off_t i; + block_off_t i, j; + unsigned diskmax = tommy_array_size(&state->diskarr); + + /* state changed */ + state->need_write = 1; /* free all the blocks of the file */ for(i=0;iblockmax;++i) { @@ -43,7 +47,18 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* if (disk->first_free_block > block_pos) disk->first_free_block = block_pos; + /* clear the block */ tommy_array_set(&disk->blockarr, block_pos, 0); + + /* invalidate the block of all the other disks */ + for(j=0;jdiskarr, j); + struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos); + if (oth_block) { + /* remove the parity info for this block */ + oth_block->flag = bit_clear(oth_block->flag, BLOCK_HAS_PARITY); + } + } } /* remove the file from the file containers */ @@ -63,6 +78,9 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* block_off_t block_max; block_off_t block_pos; + /* state changed */ + state->need_write = 1; + /* allocate the blocks of the file */ block_pos = disk->first_free_block; block_max = tommy_array_size(&disk->blockarr); @@ -119,10 +137,14 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, /* do a safety check to ensure that the common ext4 case of zeroing */ /* the size of a file after a crash doesn't propagate to the backup */ if (file->size != 0 && st->st_size == 0) { - if (!state->force_zero) { - fprintf(stderr, "The file '%s' in disk '%s' at dir '%s' has now zero size!\n", sub, disk->name, disk->dir); - fprintf(stderr, "If it's really what you want to sync it, use 'snapraid --force-zero sync\n"); - exit(EXIT_FAILURE); + /* do the check ONLY if the name is the same */ + /* otherwise it could be a deleted and recreated file */ + if (strcmp(file->sub, sub) == 0) { + if (!state->force_zero) { + fprintf(stderr, "The file '%s' in disk '%s' at dir '%s' has now zero size!\n", sub, disk->name, disk->dir); + fprintf(stderr, "If it's really what you want to sync it, use 'snapraid --force-zero sync\n"); + exit(EXIT_FAILURE); + } } } @@ -180,10 +202,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s char sub_next[PATH_MAX]; struct stat st; const char* name = dd->d_name; - int is_auto = name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)); /* skip "." and ".." files */ - if (is_auto) + if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0))) continue; pathprint(path_next, sizeof(path_next), "%s%s", dir, name); @@ -194,6 +215,12 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s exit(EXIT_FAILURE); } + /* check for not supported file names, mainly derived from the content file format */ + if (name[0] == 0 || isspace(name[0]) || isspace(name[strlen(name)-1])) { + fprintf(stderr, "Unsupported name '%s' in file '%s'\n", name, path_next); + exit(EXIT_FAILURE); + } + if (S_ISREG(st.st_mode)) { if (scan_filter(state, sub_next, name, 0) == 0) { scan_file(scan, state, disk, path_next, sub_next, &st); diff --git a/snapraid.html b/snapraid.html index a23bcfac..b0e4443a 100644 --- a/snapraid.html +++ b/snapraid.html @@ -25,7 +25,7 @@ SnapRAID is a backup program for a disk array using redundancy. SnapRAID uses a disk of the array to store redundancy information, and it allows to recover from a disk failure.

-SnapRAID is mainly targeted for a home media server, where you have +SnapRAID is mainly targeted for a home media center, where you have a lot of big files that rarely change.

The main features of SnapRAID are: @@ -51,6 +51,9 @@ failed disks. All the data in the other disks is safe. It doesn't lock-in your data. You can stop using SnapRAID at any time without the need to reformat or move data. +

  • +All your data is hashed to ensure data integrity. +

    • The official site of SnapRAID is: @@ -103,20 +106,20 @@ failure of the whole disk array.

    4 Getting Started

    -To use SnapRAID you need first to mount all the disks of your disk -array and select which one to dedicate at the redundancy information. -

    -This disk will be dedicated to this pourpuse only, and you should -not store any other data on it. +To use SnapRAID you need first select one disk of your disk array +to dedicate at the redundancy information.

    You have to pick the biggest disk in the array, as the redundancy information may grow in size as the biggest data disk in the array.

    +This disk will be dedicated to this pourpuse only and you should +not store other data on it. +

    Suppose now that you have mounted all your disks in the mount points: 

    -/mnt/diskpar
+/mnt/diskpar <- selected disk for parity
 /mnt/disk1
 /mnt/disk2
 /mnt/disk3
@@ -145,9 +148,10 @@ snapraid sync

    -This process will take some hours at best the first time, depending on -the size of the data already present in the disks. -If the disks are empty the process is immediate. +This process maty take some hours the first time, depending on the size +of the data already present in the disks. If the disks are empty +the process is immediate. +

    You can stop it at any time pressing Ctrl+C, and at the next run it will start where interrupted.

    @@ -177,6 +181,10 @@ snapraid fix Note that the fix command will revert your data at the state of the last "sync" command executed. It works like a snapshot was taken in "sync". +

    +In this regard snapraid is more like a backup program than a RAID +system. For example, you can use it to recover from an accidentally +deleted directory, simply running the fix command.

    5 Commands

    @@ -313,7 +321,7 @@ content FILE Defines the file to use to store the content of the redundancy organization. It can be placed in the same disk of the parity file, or better -in another disk, but NOT in a disk of the array. +in another disk, but NOT in a data disk of the array. This option can be used only one time. @@ -341,16 +349,16 @@ This option can be used many times.
    -block_size SIZE_IN_KILOBYTES +block_size SIZE_IN_KIBIBYTES
    -Defines the basic block size in kilo bytes of -the redundancy blocks. The default is 256 and it should -work for most conditions. -You should use this option only if you do not have enough -memory to run SnapRAID. -It requires to run TS*24/BS bytes, where TS is the total -size in bytes of your disk array, and BS is the block size -in bytes. +Defines the basic block size in kibi bytes of +the redundancy blocks. Where one kibi bytes is 1024 bytes. +The default is 256 and it should work for most conditions. +You increase this value if you do not have enough memory +to run SnapRAID. +It requires to run something about TS*24/BS bytes, where TS +is the total size in bytes of your disk array, and BS is the +block size in bytes.

    @@ -365,6 +373,13 @@ memory = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB

    +
    +You should instead decrease this value if you have a lot of +small files in the disk array. For each file, even if of few +bytes, a whole block is always allocated, so you may have a lot +of unused space. +
    +

    An example of a typical configuration is:

    @@ -386,7 +401,8 @@ Pattern Patterns are used to define the files and directories to exclude from the redundancy process.

    -It makes sense to exclude any file not worth to be saved. +It makes sense to exclude any file not worth to be saved or that +changes often.

    There are four different types of patterns: @@ -429,12 +445,11 @@ This pattern is applied only to directories and not to files.

    For example: -

     # Excludes any file named "*.bak"
 exclude *.bak
-# Excludes the root directory "lost+found"
+# Excludes the root directory "/lost+found"
 exclude /lost+found/
 # Excludes any directory named "tmp"
 exclude tmp/
@@ -444,7 +459,8 @@ exclude tmp/
 Content
 
 SnapRAID creates a content file describing the content of your disk array.
-You do not have to to modify it, altought the format of this file is
+
    
+You should never change it manually, altought the format of this file is
 described here.
 
    
 
    
@@ -459,13 +475,13 @@ defined in the configuration file.
 file DISK SIZE TIME INODE PATH
 
    
 Defines a file in the specified DISK.
-The INODE information is used to identify the file in the 'sync'
-command, allowing to move the file anywhere in disk without
+The INODE number is used to identify the file in the "sync"
+command, allowing to rename or move the file in disk without
 the need to recompute the parity for it.
 The SIZE and TIME information are used to identify if the file
-changed from the last 'sync' command, and if there is the need
+changed from the last "sync" command, and if there is the need
 to recompute the parity.
-The PATH information is used in the 'check' and 'fix' commands
+The PATH information is used in the "check" and "fix" commands
 to identify the file.
 
    
 
    
@@ -473,17 +489,31 @@ to identify the file.
    blk BLOCK HASH
    -Defines the ordered blocks used by the last defined file. -BLOCK is the block position in the parity file. +Defines the ordered parity block list used by the last defined file. +BLOCK is the block position in the "parity" file. +0 for the first block, 1 for the second one and so on. HASH is the md5 of the block. In the last block of the file, -the HASH is the hash only of the used part of the block. +the HASH is the hash of only the used part of the block.

    10 +Parity +

    +SnapRAID creates a parity file containing the parity information of your disk array. +

    +It's a binary file, containing the parity information of all the blocks defined +in the "content" file. +

    +For all the blocks at a given position, the parity information is computed with +the XOR operator applied to all the blocks. +

    +When a file block is shorter than the default block size, for example because it's +the last block of a file, it's assumed filled with 0 at the end in the XOR operation. +

    11 Copyright

    This file is Copyright (C) 2011 Andrea Mazzoleni -

    11 +

    12 See Also

    rsync(1) diff --git a/state.c b/state.c index 703852a4..6087e0d2 100644 --- a/state.c +++ b/state.c @@ -222,7 +222,7 @@ void state_read(struct snapraid_state* state) tag = s; s = strtoken(s); - if (strcmp(tag, "blk") == 0) { + if (strcmp(tag, "blk") == 0 || strcmp(tag, "inv") == 0) { char* pos; char* hash; block_off_t v_pos; @@ -265,7 +265,18 @@ void state_read(struct snapraid_state* state) fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } - block->is_hashed = 1; + + block->flag = bit_set(block->flag, BLOCK_HAS_HASH); + } + + /* set the parity only if present */ + if (tag[0] == 'b') /* blk */ + block->flag = bit_set(block->flag, BLOCK_HAS_PARITY); + + /* parity implies hash */ + if (bit_has(block->flag, BLOCK_HAS_PARITY) && !bit_has(block->flag, BLOCK_HAS_HASH)) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); } /* insert the block in the block array */ @@ -445,14 +456,21 @@ void state_write(struct snapraid_state* state) /* for each block */ for(k=0;kblockmax;++k) { struct snapraid_block* block = &file->blockvec[k]; + const char* tag; - if (block->is_hashed) { + if (bit_has(block->flag, BLOCK_HAS_PARITY)) { + tag = "blk"; + } else { + tag = "inv"; + } + + if (bit_has(block->flag, BLOCK_HAS_HASH)) { char s_hash[HASH_MAX*2+1]; strenchex(s_hash, block->hash, HASH_MAX); s_hash[HASH_MAX*2] = 0; - ret = fprintf(f, "blk %u %s\n", block->parity_pos, s_hash); + ret = fprintf(f, "%s %u %s\n", tag, block->parity_pos, s_hash); } else { - ret = fprintf(f, "blk %u\n", block->parity_pos); + ret = fprintf(f, "%s %u\n", tag, block->parity_pos); } if (ret < 0) { fprintf(stderr, "Error writing the content file '%s' in fprintf(). %s.\n", path, strerror(errno)); @@ -474,7 +492,7 @@ void state_write(struct snapraid_state* state) exit(EXIT_FAILURE); } -#if HAVE_FSYNC +#if HAVE_FSYNC if (fsync(fileno(f)) != 0) { fprintf(stderr, "Error writing the content file '%s' in fsync(). %s.\n", path, strerror(errno)); exit(EXIT_FAILURE); diff --git a/sync.c b/sync.c index 55810a6a..c7bbf9c5 100644 --- a/sync.c +++ b/sync.c @@ -58,20 +58,21 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ last = start; for(i=blockstart;iis_hashed) { - unhashed = 1; + if (block && !bit_has(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) { + one_invalid = 1; break; } } - if (!unhashed) + /* if no invalid block skip */ + if (!one_invalid) continue; /* start with 0 */ @@ -141,7 +142,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ md5_update(&md5, block_buffer, read_size); md5_final(&md5, hash); - if (block->is_hashed) { + if (bit_has(block->flag, BLOCK_HAS_HASH)) { /* compare the hash */ if (memcmp(hash, block->hash, HASH_MAX) != 0) { fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); @@ -179,7 +180,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ if (!block) continue; - block->is_hashed = 1; + block->flag = bit_set(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY); } /* mark the state as needing write */