rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 02:49:02 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Handles correctly file deletion in sync.

Browse source
This commit is contained in:
Andrea Mazzoleni 2011-03-22 23:23:45 +01:00
parent c6ab503395
commit 0451c9e8a0
7 changed files with 146 additions and 65 deletions
Download patch file Download diff file Expand all files Collapse all files

22
check.c
View file

@ -62,23 +62,22 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
last = start;
for(i=blockstart;i<blockmax;++i) {
int hashed;
int failed;
struct snapraid_block* failed_block;
struct snapraid_handle* failed_handle;
int one_hash;
/* for each disk, search for a hashed block */
hashed = 0;
/* for each disk */
one_hash = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block && block->is_hashed) {
hashed = 1;
break;
if (block && bit_has(block->flag, BLOCK_HAS_HASH)) {
one_hash = 1;
}
}
/* if there is at least an hashed block we have to check */
if (!hashed)
/* if no hashed block skip */
if (!one_hash)
continue;
/* start with 0 */
@ -98,8 +97,10 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
if (!block)
continue;
/* we can check and fix only if the block is hashed */
if (!block->is_hashed)
/* we try to check and fix only if the block is hashed */
/* it could be that a file was added, and not yet synched */
/* so we should not include not hashed block in the parity computation */
if (!bit_has(block->flag, BLOCK_HAS_HASH))
continue;
ret = handle_close_if_different(&handle[j], block->file);
@ -385,3 +386,4 @@ void state_check(struct snapraid_state* state, int fix, block_off_t blockstart)
if (unrecoverable_error != 0)
exit(EXIT_FAILURE);
}

2
elem.c
View file

@ -145,7 +145,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t
/* set the back pointer */
for(i=0;i<file->blockmax;++i) {
file->blockvec[i].parity_pos = POS_INVALID;
file->blockvec[i].is_hashed = 0;
file->blockvec[i].flag = 0;
file->blockvec[i].file = file;
}

5
elem.h
View file

@ -60,12 +60,15 @@ struct snapraid_filter {
struct snapraid_file;
#define BLOCK_HAS_HASH 1 /**< If the hash value is valid. */
#define BLOCK_HAS_PARITY 2 /**< If the parity block is valid. */
/**
* Block of a file.
*/
struct snapraid_block {
block_off_t parity_pos; /**< Position of the block in the parity. */
int is_hashed; /**< If the hash of the block is valid. */
unsigned flag; /**< If the hash of the block is valid. */
struct snapraid_file* file; /**< Back pointer to the file owning this block. */
unsigned char hash[HASH_MAX]; /**< Hash of the block. */
};

41
scan.c
View file

@ -33,7 +33,11 @@ struct snapraid_scan {
*/
static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
block_off_t i;
block_off_t i, j;
unsigned diskmax = tommy_array_size(&state->diskarr);
/* state changed */
state->need_write = 1;
/* free all the blocks of the file */
for(i=0;i<file->blockmax;++i) {
@ -43,7 +47,18 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
if (disk->first_free_block > block_pos)
disk->first_free_block = block_pos;
/* clear the block */
tommy_array_set(&disk->blockarr, block_pos, 0);
/* invalidate the block of all the other disks */
for(j=0;j<diskmax;++j) {
struct snapraid_disk* oth_disk = tommy_array_get(&state->diskarr, j);
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
if (oth_block) {
/* remove the parity info for this block */
oth_block->flag = bit_clear(oth_block->flag, BLOCK_HAS_PARITY);
}
}
}
/* remove the file from the file containers */
@ -63,6 +78,9 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
block_off_t block_max;
block_off_t block_pos;
/* state changed */
state->need_write = 1;
/* allocate the blocks of the file */
block_pos = disk->first_free_block;
block_max = tommy_array_size(&disk->blockarr);
@ -119,10 +137,14 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
/* do a safety check to ensure that the common ext4 case of zeroing */
/* the size of a file after a crash doesn't propagate to the backup */
if (file->size != 0 && st->st_size == 0) {
if (!state->force_zero) {
fprintf(stderr, "The file '%s' in disk '%s' at dir '%s' has now zero size!\n", sub, disk->name, disk->dir);
fprintf(stderr, "If it's really what you want to sync it, use 'snapraid --force-zero sync\n");
exit(EXIT_FAILURE);
/* do the check ONLY if the name is the same */
/* otherwise it could be a deleted and recreated file */
if (strcmp(file->sub, sub) == 0) {
if (!state->force_zero) {
fprintf(stderr, "The file '%s' in disk '%s' at dir '%s' has now zero size!\n", sub, disk->name, disk->dir);
fprintf(stderr, "If it's really what you want to sync it, use 'snapraid --force-zero sync\n");
exit(EXIT_FAILURE);
}
}
}
@ -180,10 +202,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s
char sub_next[PATH_MAX];
struct stat st;
const char* name = dd->d_name;
int is_auto = name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0));
/* skip "." and ".." files */
if (is_auto)
if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
continue;
pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
@ -194,6 +215,12 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s
exit(EXIT_FAILURE);
}
/* check for not supported file names, mainly derived from the content file format */
if (name[0] == 0 || isspace(name[0]) || isspace(name[strlen(name)-1])) {
fprintf(stderr, "Unsupported name '%s' in file '%s'\n", name, path_next);
exit(EXIT_FAILURE);
}
if (S_ISREG(st.st_mode)) {
if (scan_filter(state, sub_next, name, 0) == 0) {
scan_file(scan, state, disk, path_next, sub_next, &st);

94
snapraid.html
View file

@ -25,7 +25,7 @@ SnapRAID is a backup program for a disk array using redundancy.
SnapRAID uses a disk of the array to store redundancy information,
and it allows to recover from a disk failure.
<p>
SnapRAID is mainly targeted for a home media server, where you have
SnapRAID is mainly targeted for a home media center, where you have
a lot of big files that rarely change.
<p>
The main features of SnapRAID are:
@ -51,6 +51,9 @@ failed disks. All the data in the other disks is safe.
It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data.
</li>
<li>
All your data is hashed to ensure data integrity.
</li>
</ul>
<p>
The official site of SnapRAID is:
@ -103,20 +106,20 @@ failure of the whole disk array.
<h2><a name="4">4</a>
Getting Started
</h2>
To use SnapRAID you need first to mount all the disks of your disk
array and select which one to dedicate at the redundancy information.
<p>
This disk will be dedicated to this pourpuse only, and you should
not store any other data on it.
To use SnapRAID you need first select one disk of your disk array
to dedicate at the redundancy information.
<p>
You have to pick the biggest disk in the array, as the redundancy
information may grow in size as the biggest data disk in the array.
<p>
This disk will be dedicated to this pourpuse only and you should
not store other data on it.
<p>
Suppose now that you have mounted all your disks in the mount points:
<p>
<table width="100%" border="0" cellspacing="0" cellpadding="2"><tr><td width="5%"></td><td width="95%">
<pre>
/mnt/diskpar
/mnt/diskpar &lt;- selected disk for parity
/mnt/disk1
/mnt/disk2
/mnt/disk3
@ -145,9 +148,10 @@ snapraid sync
</pre>
</td></tr></table>
<p>
This process will take some hours at best the first time, depending on
the size of the data already present in the disks.
If the disks are empty the process is immediate.
This process maty take some hours the first time, depending on the size
of the data already present in the disks. If the disks are empty
the process is immediate.
<p>
You can stop it at any time pressing Ctrl+C, and at the next run it
will start where interrupted.
<p>
@ -177,6 +181,10 @@ snapraid fix
Note that the fix command will revert your data at the state of the
last "sync" command executed. It works like a snapshot was taken
in "sync".
<p>
In this regard snapraid is more like a backup program than a RAID
system. For example, you can use it to recover from an accidentally
deleted directory, simply running the fix command.
<h2><a name="5">5</a>
Commands
</h2>
@ -313,7 +321,7 @@ content FILE
Defines the file to use to store the content of the redundancy
organization.
It can be placed in the same disk of the parity file, or better
in another disk, but NOT in a disk of the array.
in another disk, but NOT in a data disk of the array.
This option can be used only one time.
</td></tr>
</table>
@ -341,16 +349,16 @@ This option can be used many times.
</table>
<table width="100%" border="0" cellspacing="0" cellpadding="2">
<tr valign="top" align="left"><td><strong>
block_size SIZE_IN_KILOBYTES
block_size SIZE_IN_KIBIBYTES
</strong></td></tr><tr><td>
Defines the basic block size in kilo bytes of
the redundancy blocks. The default is 256 and it should
work for most conditions.
You should use this option only if you do not have enough
memory to run SnapRAID.
It requires to run TS*24/BS bytes, where TS is the total
size in bytes of your disk array, and BS is the block size
in bytes.
Defines the basic block size in kibi bytes of
the redundancy blocks. Where one kibi bytes is 1024 bytes.
The default is 256 and it should work for most conditions.
You increase this value if you do not have enough memory
to run SnapRAID.
It requires to run something about TS*24/BS bytes, where TS
is the total size in bytes of your disk array, and BS is the
block size in bytes.
</td></tr>
</table>
<p>
@ -365,6 +373,13 @@ memory = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB
</pre>
</td></tr></table>
<p>
<table width="100%" border="0" cellspacing="0" cellpadding="2"><tr><td width="5%"></td><td width="95%">
You should instead decrease this value if you have a lot of
small files in the disk array. For each file, even if of few
bytes, a whole block is always allocated, so you may have a lot
of unused space.
</td></tr></table>
<p>
An example of a typical configuration is:
<p>
<table width="100%" border="0" cellspacing="0" cellpadding="2"><tr><td width="5%"></td><td width="95%">
@ -386,7 +401,8 @@ Pattern
Patterns are used to define the files and directories to exclude
from the redundancy process.
<p>
It makes sense to exclude any file not worth to be saved.
It makes sense to exclude any file not worth to be saved or that
changes often.
<p>
There are four different types of patterns:
<table width="100%" border="0" cellspacing="0" cellpadding="2">
@ -429,12 +445,11 @@ This pattern is applied only to directories and not to files.
</table>
<p>
For example:
<p>
<table width="100%" border="0" cellspacing="0" cellpadding="2"><tr><td width="5%"></td><td width="95%">
<pre>
# Excludes any file named "*.bak"
exclude *.bak
# Excludes the root directory "lost+found"
# Excludes the root directory "/lost+found"
exclude /lost+found/
# Excludes any directory named "tmp"
exclude tmp/
@ -444,7 +459,8 @@ exclude tmp/
Content
</h2>
SnapRAID creates a content file describing the content of your disk array.
You do not have to to modify it, altought the format of this file is
<p>
You should never change it manually, altought the format of this file is
described here.
<table width="100%" border="0" cellspacing="0" cellpadding="2">
<tr valign="top" align="left"><td><strong>
@ -459,13 +475,13 @@ defined in the configuration file.
file DISK SIZE TIME INODE PATH
</strong></td></tr><tr><td>
Defines a file in the specified DISK.
The INODE information is used to identify the file in the 'sync'
command, allowing to move the file anywhere in disk without
The INODE number is used to identify the file in the "sync"
command, allowing to rename or move the file in disk without
the need to recompute the parity for it.
The SIZE and TIME information are used to identify if the file
changed from the last 'sync' command, and if there is the need
changed from the last "sync" command, and if there is the need
to recompute the parity.
The PATH information is used in the 'check' and 'fix' commands
The PATH information is used in the "check" and "fix" commands
to identify the file.
</td></tr>
</table>
@ -473,17 +489,31 @@ to identify the file.
<tr valign="top" align="left"><td><strong>
blk BLOCK HASH
</strong></td></tr><tr><td>
Defines the ordered blocks used by the last defined file.
BLOCK is the block position in the parity file.
Defines the ordered parity block list used by the last defined file.
BLOCK is the block position in the "parity" file.
0 for the first block, 1 for the second one and so on.
HASH is the md5 of the block. In the last block of the file,
the HASH is the hash only of the used part of the block.
the HASH is the hash of only the used part of the block.
</td></tr>
</table>
<h2><a name="10">10</a>
Parity
</h2>
SnapRAID creates a parity file containing the parity information of your disk array.
<p>
It's a binary file, containing the parity information of all the blocks defined
in the "content" file.
<p>
For all the blocks at a given position, the parity information is computed with
the XOR operator applied to all the blocks.
<p>
When a file block is shorter than the default block size, for example because it's
the last block of a file, it's assumed filled with 0 at the end in the XOR operation.
<h2><a name="11">11</a>
Copyright
</h2>
This file is Copyright (C) 2011 Andrea Mazzoleni
<h2><a name="11">11</a>
<h2><a name="12">12</a>
See Also
</h2>
rsync(1)

30
state.c
View file

@ -222,7 +222,7 @@ void state_read(struct snapraid_state* state)
tag = s;
s = strtoken(s);
if (strcmp(tag, "blk") == 0) {
if (strcmp(tag, "blk") == 0 || strcmp(tag, "inv") == 0) {
char* pos;
char* hash;
block_off_t v_pos;
@ -265,7 +265,18 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
block->is_hashed = 1;
block->flag = bit_set(block->flag, BLOCK_HAS_HASH);
}
/* set the parity only if present */
if (tag[0] == 'b') /* blk */
block->flag = bit_set(block->flag, BLOCK_HAS_PARITY);
/* parity implies hash */
if (bit_has(block->flag, BLOCK_HAS_PARITY) && !bit_has(block->flag, BLOCK_HAS_HASH)) {
fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
/* insert the block in the block array */
@ -445,14 +456,21 @@ void state_write(struct snapraid_state* state)
/* for each block */
for(k=0;k<file->blockmax;++k) {
struct snapraid_block* block = &file->blockvec[k];
const char* tag;
if (block->is_hashed) {
if (bit_has(block->flag, BLOCK_HAS_PARITY)) {
tag = "blk";
} else {
tag = "inv";
}
if (bit_has(block->flag, BLOCK_HAS_HASH)) {
char s_hash[HASH_MAX*2+1];
strenchex(s_hash, block->hash, HASH_MAX);
s_hash[HASH_MAX*2] = 0;
ret = fprintf(f, "blk %u %s\n", block->parity_pos, s_hash);
ret = fprintf(f, "%s %u %s\n", tag, block->parity_pos, s_hash);
} else {
ret = fprintf(f, "blk %u\n", block->parity_pos);
ret = fprintf(f, "%s %u\n", tag, block->parity_pos);
}
if (ret < 0) {
fprintf(stderr, "Error writing the content file '%s' in fprintf(). %s.\n", path, strerror(errno));
@ -474,7 +492,7 @@ void state_write(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
#if HAVE_FSYNC
#if HAVE_FSYNC
if (fsync(fileno(f)) != 0) {
fprintf(stderr, "Error writing the content file '%s' in fsync(). %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);

17
sync.c
View file

@ -58,20 +58,21 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
last = start;
for(i=blockstart;i<blockmax;++i) {
int unhashed;
int one_invalid;
int ret;
/* for each disk, search for an unhashed block */
unhashed = 0;
/* for each disk */
one_invalid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block && !block->is_hashed) {
unhashed = 1;
if (block && !bit_has(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
one_invalid = 1;
break;
}
}
if (!unhashed)
/* if no invalid block skip */
if (!one_invalid)
continue;
/* start with 0 */
@ -141,7 +142,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
md5_update(&md5, block_buffer, read_size);
md5_final(&md5, hash);
if (block->is_hashed) {
if (bit_has(block->flag, BLOCK_HAS_HASH)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_MAX) != 0) {
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
@ -179,7 +180,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
if (!block)
continue;
block->is_hashed = 1;
block->flag = bit_set(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY);
}
/* mark the state as needing write */