rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added a checksum to the content file.

Browse source
This commit is contained in:
Andrea Mazzoleni 2011-12-31 12:09:05 +01:00
parent cd906d5dac
commit a0ad57d4f1
8 changed files with 210 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

1
HISTORY
View file

@ -8,6 +8,7 @@ SnapRAID HISTORY
* Faster and parallel writing of content files.
* The example configuration files now put the content files in the data
disks instead than in the parity disks.
* Added a checksum at the content file to ensure its integrity.
1.7 2011/11
===========

5
TODO
View file

@ -12,14 +12,9 @@ the process making the create() operation to fail.
* If a directory exists with the same name of a parity/content file be more explicative
on the error message. See: https://sourceforge.net/projects/snapraid/forums/forum/1677233/topic/4861034
Also adding an estension to these files in the configuration file for Windows may help to understand
that they are files and not directories.
* Easy way to find files using a specified block.
* Adds a checksum for the content file. When reading if the file is invalid,
skip to another.
- v1.9
* Allow to filter the files to check/fix based on the drive where they reside.

14
snapraid.1
View file

@ -188,22 +188,22 @@ disk d3 /mnt/disk3/
.RE
.PP
If you are in Windows, you should use drive letters and backslashes
instead of slashes:
instead of slashes, and if you like, also file extensions.
.PP
.RS 4
parity E:\(rspar\(rsparity
parity E:\(rspar\(rsparity.par
.PD 0
.PP
.PD
content C:\(rssnapraid\(rscontent
content C:\(rssnapraid\(rscontent.lst
.PD 0
.PP
.PD
content F:\(rsarray\(rscontent
content F:\(rsarray\(rscontent.lst
.PD 0
.PP
.PD
content G:\(rsarray\(rscontent
content G:\(rsarray\(rscontent.lst
.PD 0
.PP
.PD
@ -826,6 +826,10 @@ defined in the configuration file.
Defines the checksum kind used. It can be \(a"md5\(a" or \(a"murmur3\(a".
.SS map NAME INDEX
Defines the position INDEX of the disk NAME in the parity computation.
.SS sign SIGN
Signature checksum of the content file to ensure that it doesn\(cqt get
corrupted. If you want to modify the content file manually, you have
to remove this line to avoid this check.
.SS file DISK SIZE TIME INODE PATH
Defines a file in the specified DISK.
.PP

10
snapraid.conf.example.windows
View file

@ -3,13 +3,13 @@
# Defines the file to use as Parity storage
# It must NOT be in a data disk
# Format: "parity FILE_PATH"
parity E:\par\parity
parity E:\par\parity.par
# Defines the file to use as Q-Parity storage
# If specified, it enables a double failures protection like RAID6
# It must NOT be in a data disk
# Format: "q-parity FILE_PATH"
#q-parity F:\qar\q-parity
#q-parity F:\qar\q-parity.par
# Defines the files to use as content list
# You can use multiple specification to store more copies
@ -17,9 +17,9 @@ parity E:\par\parity
# They can be in the disks used for data, parity or boot,
# but each file must be in a different disk
# Format: "content FILE_PATH"
content C:\snapraid\content
content G:\array\content
content H:\array\content
content C:\snapraid\content.lst
content G:\array\content.lst
content H:\array\content.lst
# Defines the data disks to use
# The order is relevant for parity, do not change it

15
snapraid.d
View file

@ -114,12 +114,12 @@ Getting Started
:disk d3 /mnt/disk3/
If you are in Windows, you should use drive letters and backslashes
instead of slashes:
instead of slashes, and if you like, also file extensions.
:parity E:\par\parity
:content C:\snapraid\content
:content F:\array\content
:content G:\array\content
:parity E:\par\parity.par
:content C:\snapraid\content.lst
:content F:\array\content.lst
:content G:\array\content.lst
:disk d1 F:\array\
:disk d2 G:\array\
:disk d3 H:\array\
@ -581,6 +581,11 @@ Content
map NAME INDEX
Defines the position INDEX of the disk NAME in the parity computation.
sign SIGN
Signature checksum of the content file to ensure that it doesn't get
corrupted. If you want to modify the content file manually, you have
to remove this line to avoid this check.
file DISK SIZE TIME INODE PATH
Defines a file in the specified DISK.

27
snapraid.txt
View file

@ -32,14 +32,14 @@ features of SnapRAID are:
* You can start using SnapRAID with already filled disks.
* The disks can have different sizes.
* You can add more disks at any time.
* You can add disks at any time.
* If you accidentally delete some files in a disk, you can
recover them.
* If more than two disks fail, you lose the data only on the
failed disks. All the data in the other disks is safe.
* It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data.
* All your data is hashed to ensure data integrity and avoid
* All your data is hashed to ensure data integrity and to avoid
silent corruption.
The official site of SnapRAID is:
@ -127,12 +127,12 @@ the following options:
disk d3 /mnt/disk3/
If you are in Windows, you should use drive letters and backslashes
instead of slashes:
instead of slashes, and if you like, also file extensions.
parity E:\par\parity
content C:\snapraid\content
content F:\array\content
content G:\array\content
parity E:\par\parity.par
content C:\snapraid\content.lst
content F:\array\content.lst
content G:\array\content.lst
disk d1 F:\array\
disk d2 G:\array\
disk d3 H:\array\
@ -644,7 +644,14 @@ Defines the checksum kind used. It can be "md5" or "murmur3".
Defines the position INDEX of the disk NAME in the parity computation.
10.4 file DISK SIZE TIME INODE PATH
10.4 sign SIGN
--------------
Signature checksum of the content file to ensure that it doesn't get
corrupted. If you want to modify the content file manually, you have
to remove this line to avoid this check.
10.5 file DISK SIZE TIME INODE PATH
-----------------------------------
Defines a file in the specified DISK.
@ -660,7 +667,7 @@ to recompute the parity.
The PATH information is used in the "check" and "fix" commands
to identify the file.
10.5 blk BLOCK HASH
10.6 blk BLOCK HASH
-------------------
Defines the ordered parity block list used by the last defined file.
@ -671,7 +678,7 @@ BLOCK is the block position in the "parity" file.
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
10.6 inv BLOCK [HASH]
10.7 inv BLOCK [HASH]
---------------------
Like "blk", but inform that the parity of this block is invalid.

97
state.c
View file

@ -531,10 +531,12 @@ void state_read(struct snapraid_state* state)
unsigned count_block;
unsigned count_link;
tommy_node* node;
oathash_t hash;
count_file = 0;
count_block = 0;
count_link = 0;
hash = 0;
/* iterate over all the available content files and load the first one present */
f = 0;
@ -634,7 +636,16 @@ void state_read(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
/* set the parity only if present */
if (tag[0] == 'b') {
block_flag_set(block, BLOCK_HAS_PARITY);
hash = oathash8(hash, 'b');
} else {
hash = oathash8(hash, 'i');
}
block->parity_pos = v_pos;
hash = oathash32(hash, v_pos);
/* check if we are at the end of the line */
c = sgeteol(f);
@ -653,14 +664,11 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashm(hash, block->hash, HASH_SIZE);
block_flag_set(block, BLOCK_HAS_HASH);
}
/* set the parity only if present */
if (strcmp(tag, "blk") == 0)
block_flag_set(block, BLOCK_HAS_PARITY);
/* parity implies hash */
if (block_flag_has(block, BLOCK_HAS_PARITY) && !block_flag_has(block, BLOCK_HAS_HASH)) {
fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
@ -688,6 +696,8 @@ void state_read(struct snapraid_state* state)
uint64_t v_inode;
tommy_node* i;
hash = oathash8(hash, 'f');
if (file) {
fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
@ -698,6 +708,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, buffer);
c = sgetc(f);
if (c != ' ') {
@ -710,6 +721,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash64(hash, v_size);
c = sgetc(f);
if (c != ' ') {
@ -722,6 +734,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash64(hash, v_mtime);
c = sgetc(f);
if (c != ' ') {
@ -734,6 +747,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash64(hash, v_inode);
c = sgetc(f);
if (c != ' ') {
@ -751,6 +765,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, sub);
/* find the disk */
for(i=state->disklist;i!=0;i=i->next) {
@ -790,11 +805,14 @@ void state_read(struct snapraid_state* state)
tommy_node* i;
struct snapraid_link* link;
hash = oathash8(hash, 's');
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'symlink' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, buffer);
c = sgetc(f);
if (c != ' ') {
@ -807,6 +825,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'symlink' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, sub);
c = sgeteol(f);
if (c != '\n') {
@ -836,6 +855,7 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'symlink' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, linkto);
if (!*sub || !*linkto) {
fprintf(stderr, "Invalid 'symlink' specification in '%s' at line %u\n", path, line);
@ -863,6 +883,8 @@ void state_read(struct snapraid_state* state)
/* stat */
++count_link;
} else if (strcmp(tag, "checksum") == 0) {
hash = oathash8(hash, 'c');
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'checksum' specification in '%s' at line %u\n", path, line);
@ -871,8 +893,10 @@ void state_read(struct snapraid_state* state)
if (strcmp(buffer, "md5") == 0) {
state->hash = HASH_MD5;
hash = oathash8(hash, 'm');
} else if (strcmp(buffer, "murmur3") == 0) {
state->hash = HASH_MURMUR3;
hash = oathash8(hash, 'u');
} else {
fprintf(stderr, "Invalid 'checksum' specification '%s' in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
@ -880,11 +904,14 @@ void state_read(struct snapraid_state* state)
} else if (strcmp(tag, "blksize") == 0) {
block_off_t blksize;
hash = oathash8(hash, 'z');
ret = sgetu32(f, &blksize);
if (ret < 0) {
fprintf(stderr, "Invalid 'blksize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, blksize);
if (blksize != state->block_size) {
fprintf(stderr, "Mismatching 'blksize' and 'block_size' specification in '%s' at line %u\n", path, line);
@ -895,11 +922,14 @@ void state_read(struct snapraid_state* state)
struct snapraid_map* map;
uint32_t v_pos;
hash = oathash8(hash, 'm');
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, buffer);
c = sgetc(f);
if (c != ' ') {
@ -912,10 +942,25 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, v_pos);
map = map_alloc(buffer, v_pos);
tommy_list_insert_tail(&state->maplist, &map->node, map);
} else if (strcmp(tag, "sign") == 0) {
oathash_t sign;
ret = sgetu32(f, &sign);
if (ret < 0) {
fprintf(stderr, "Invalid 'sign' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
if (sign != hash) {
fprintf(stderr, "Mismatching 'sign' in '%s' at line %u\n", path, line);
fprintf(stderr, "Likely this content file is damaged. Use an alternate copy\n");
exit(EXIT_FAILURE);
}
} else if (tag[0] == 0) {
/* allow empty lines */
sgetspace(f);
@ -975,10 +1020,12 @@ void state_write(struct snapraid_state* state)
unsigned count_content;
tommy_node* i;
unsigned k;
oathash_t hash;
count_file = 0;
count_block = 0;
count_link = 0;
hash = 0;
/* count the content files */
count_content = 0;
@ -1012,17 +1059,24 @@ void state_write(struct snapraid_state* state)
}
sputsl("blksize ", f);
hash = oathash8(hash, 'z');
sputu32(state->block_size, f);
hash = oathash32(hash, state->block_size);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
}
if (state->hash == HASH_MD5)
if (state->hash == HASH_MD5) {
sputsl("checksum md5", f);
else
hash = oathash8(hash, 'c');
hash = oathash8(hash, 'm');
} else {
sputsl("checksum murmur3", f);
hash = oathash8(hash, 'c');
hash = oathash8(hash, 'u');
}
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -1033,9 +1087,12 @@ void state_write(struct snapraid_state* state)
for(i=state->maplist;i!=0;i=i->next) {
struct snapraid_map* map = i->data;
sputsl("map ", f);
hash = oathash8(hash, 'm');
sputs(map->name, f);
hash = oathashs(hash, map->name);
sputc(' ', f);
sputu32(map->position, f);
hash = oathash32(hash, map->position);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -1061,15 +1118,21 @@ void state_write(struct snapraid_state* state)
inode = file->inode;
sputsl("file ", f);
hash = oathash8(hash, 'f');
sputs(disk->name, f);
hash = oathashs(hash, disk->name);
sputc(' ', f);
sputu64(size, f);
hash = oathash64(hash, size);
sputc(' ', f);
sputu64(mtime, f);
hash = oathash64(hash, mtime);
sputc(' ', f);
sputu64(inode, f);
hash = oathash64(hash, inode);
sputc(' ', f);
sputs(file->sub, f);
hash = oathashs(hash, file->sub);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -1082,17 +1145,21 @@ void state_write(struct snapraid_state* state)
if (block_flag_has(block, BLOCK_HAS_PARITY)) {
sputsl("blk ", f);
hash = oathash8(hash, 'b');
} else {
sputsl("inv ", f);
hash = oathash8(hash, 'i');
}
sputu32(block->parity_pos, f);
hash = oathash32(hash, block->parity_pos);
if (block_flag_has(block, BLOCK_HAS_HASH)) {
sputu32(block->parity_pos, f);
sputc(' ', f);
sputhex(block->hash, HASH_SIZE, f);
} else {
sputu32(block->parity_pos, f);
hash = oathashm(hash, block->hash, HASH_SIZE);
}
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -1110,12 +1177,16 @@ void state_write(struct snapraid_state* state)
struct snapraid_link* link = j->data;
sputsl("symlink ", f);
hash = oathash8(hash, 's');
sputs(disk->name, f);
hash = oathashs(hash, disk->name);
sputc(' ', f);
sputs(link->sub, f);
hash = oathashs(hash, link->sub);
sputeol(f);
sputsl("to ", f);
sputs(link->linkto, f);
hash = oathashs(hash, link->linkto);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -1126,6 +1197,14 @@ void state_write(struct snapraid_state* state)
}
}
sputsl("sign ", f);
sputu32(hash, f);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
}
/* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */
/* than even in a system crash event we have one valid copy of the file. */

80
util.h
View file

@ -369,5 +369,85 @@ void* malloc_nofail_align(size_t size, void** freeptr);
*/
void memhash(unsigned kind, void* digest, const void* src, unsigned size);
/****************************************************************************/
/* sax */
/**
* One-at-a-time hash type.
* This is a really fast, but insecure hash.
* This is only *partially* based on the Bob Jenkins one-at-a-time hash.
*/
typedef uint32_t oathash_t;
/**
* Hash a byte.
*/
static inline oathash_t oathash8(oathash_t hash, unsigned char value)
{
hash += value;
hash += hash << 10;
hash ^= hash >> 6;
return hash;
}
/**
* Hash a 32 bits word.
*/
static inline oathash_t oathash32(oathash_t hash, uint32_t value)
{
hash += value;
hash += hash << 10;
hash ^= hash >> 6;
return hash;
}
/**
* Hash a 64 bits word.
* \note Based on one-at-a-time by Bob Jenkins.
*/
static inline oathash_t oathash64(oathash_t hash, uint64_t value)
{
hash += value & 0xFFFFFFFF;
hash += hash << 10;
hash ^= hash >> 6;
hash += value >> 32;
hash += hash << 10;
hash ^= hash >> 6;
return hash;
}
/**
* Hash a string.
*/
static inline oathash_t oathashs(oathash_t hash, const char* value)
{
while (*value) {
hash += *value;
hash += hash << 10;
hash ^= hash >> 6;
++value;
}
return hash;
}
/**
* Hash a range of memory.
* The size must be a 32 bits multiplier.
*/
static inline oathash_t oathashm(oathash_t hash, const unsigned char* value, unsigned size)
{
assert(size % 4 == 0);
size /= 4;
while (size--) {
uint32_t w32 = value[0] | ((unsigned)value[1] << 8) | ((unsigned)value[2] << 16) | ((unsigned)value[3] << 24);
hash += w32;
hash += hash << 10;
hash ^= hash >> 6;
value += 4;
}
return hash;
}
#endif