diff --git a/scan.c b/scan.c index 2c550134..aa5ee5c5 100644 --- a/scan.c +++ b/scan.c @@ -108,6 +108,14 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, file->is_present = 1; return; } else { + /* do a safety check to ensure that the common ext4 case of zeroing */ + /* the size of a file after a crash doesn't propagate to the backup */ + if (file->size != 0 && st->st_size == 0) { + fprintf(stderr, "The file '%s' in disk '%s' at dir '%s' has zero size!\n", sub, disk->name, disk->dir); + fprintf(stderr, "If it's really what you want to do, use 'snapraid --force sync\n"); + exit(EXIT_FAILURE); + } + /* remove and reinsert it */ ++scan->count_change; --scan->count_insert; @@ -241,7 +249,7 @@ void state_scan(struct snapraid_state* state) if (scan[i].count_equal == 0 && scan[i].count_remove != 0) { if (!state->force) { fprintf(stderr, "All the file in disk '%s' at dir '%s' are missing!\n", disk->name, disk->dir); - fprintf(stderr, "If it's really what you want to do, use 'snapraid -f sync\n"); + fprintf(stderr, "If it's really what you want to do, use 'snapraid --force sync\n"); exit(EXIT_FAILURE); } } diff --git a/snapraid.conf.example b/snapraid.conf.example index ed6c8e7c..29abdb8b 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -31,6 +31,6 @@ exclude /lost+found/ # Define the block size in kilo bytes (1024 bytes). # Format: "block_size SIZE_IN_KB" -# Default value is 128 -> 128 kilo bytes -block_size 128 +# Default value is 256 -> 256 kilo bytes +block_size 256 diff --git a/snapraid.d b/snapraid.d index b41a03ba..f1f93dc2 100644 --- a/snapraid.d +++ b/snapraid.d @@ -154,7 +154,7 @@ Configuration =block_size SIZE_IN_KILOBYTES Defines the basic block size in kilo bytes of - the redundancy blocks. The default is 128 and it should + the redundancy blocks. The default is 256 and it should work for most conditions. You should use this option only if you do not have enough memory to run SnapRAID. @@ -162,10 +162,10 @@ Configuration size in bytes of your disk array, and BS is the block size in bytes. - For example with 6 disk of 2 TB and a block size of 128 kB you + For example with 6 disk of 2 TB and a block size of 256 kB you have: - :memory = (6 * 2 * 2^40) * 24 / (128 * 2^10) = 2.3 GB + :memory = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GB An example of a typical configuration is: diff --git a/snapraid.html b/snapraid.html index ac625d2d..452145eb 100644 --- a/snapraid.html +++ b/snapraid.html @@ -232,7 +232,7 @@ This option can be used many times. block_size SIZE_IN_KILOBYTES Defines the basic block size in kilo bytes of -the redundancy blocks. The default is 128 and it should +the redundancy blocks. The default is 256 and it should work for most conditions. You should use this option only if you do not have enough memory to run SnapRAID. @@ -243,13 +243,13 @@ in bytes.

-For example with 6 disk of 2 TB and a block size of 128 kB you +For example with 6 disk of 2 TB and a block size of 256 kB you have: 

-memory = (6 * 2 * 2^40) * 24 / (128 * 2^10) = 2.3 GB
+memory = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GB

diff --git a/snapraid.txt b/snapraid.txt index fb70cd74..ed867f5f 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -170,7 +170,7 @@ it should contains the following options: block_size SIZE_IN_KILOBYTES Defines the basic block size in kilo bytes of - the redundancy blocks. The default is 128 and it should + the redundancy blocks. The default is 256 and it should work for most conditions. You should use this option only if you do not have enough memory to run SnapRAID. @@ -178,10 +178,10 @@ it should contains the following options: size in bytes of your disk array, and BS is the block size in bytes. - For example with 6 disk of 2 TB and a block size of 128 kB you + For example with 6 disk of 2 TB and a block size of 256 kB you have: - memory = (6 * 2 * 2^40) * 24 / (128 * 2^10) = 2.3 GB + memory = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GB An example of a typical configuration is: diff --git a/state.c b/state.c index fd0a3259..5559a94f 100644 --- a/state.c +++ b/state.c @@ -29,7 +29,7 @@ void state_init(struct snapraid_state* state) { state->verbose = 0; - state->block_size = 128*1024; /* default 128 kB */ + state->block_size = 256 * 1024; /* default 256 kB */ state->content[0] = 0; state->parity[0] = 0; tommy_array_init(&state->diskarr); @@ -166,7 +166,7 @@ void state_read(struct snapraid_state* state) char path[PATH_MAX]; struct snapraid_disk* disk; struct snapraid_file* file; - unsigned block_index; + block_off_t blockidx; unsigned line; unsigned count_file; unsigned count_block; @@ -190,7 +190,7 @@ void state_read(struct snapraid_state* state) disk = 0; file = 0; line = 0; - block_index = 0; + blockidx = 0; while (1) { char buffer[TEXT_LINE_MAX]; char* tag; @@ -218,7 +218,66 @@ void state_read(struct snapraid_state* state) tag = s; s = strtoken(s); - if (strcmp(tag, "file") == 0) { + if (strcmp(tag, "blk") == 0) { + char* pos; + char* hash; + block_off_t v_pos; + struct snapraid_block* block; + char* e; + + if (!file) { + fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + pos = s; + s = strtoken(s); + hash = s; + + ret = stru32(pos, &v_pos); + if (ret != 0) { + fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + if (blockidx >= file->blockmax) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + block = &file->blockvec[blockidx]; + + if (block->parity_pos != POS_INVALID) { + fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + block->parity_pos = v_pos; + + /* set the hash only if present */ + if (*hash != 0) { + e = strdechex(block->hash, HASH_MAX, hash); + if (e) { + fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + block->is_hashed = 1; + } + + /* insert the block in the block array */ + tommy_array_grow(&disk->blockarr, block->parity_pos + 1); + tommy_array_set(&disk->blockarr, block->parity_pos, block); + + /* check for termination of the block list */ + ++blockidx; + if (blockidx == file->blockmax) { + file = 0; + disk = 0; + } + + /* stat */ + ++count_block; + } else if (strcmp(tag, "file") == 0) { char* name; char* size; char* mtime; @@ -276,75 +335,30 @@ void state_read(struct snapraid_state* state) tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); /* start the block allocation of the file */ - block_index = 0; + blockidx = 0; /* check for empty file */ - if (block_index == file->blockmax) { + if (blockidx == file->blockmax) { file = 0; disk = 0; } /* stat */ ++count_file; - } else if (strcmp(tag, "blk") == 0) { - char* pos; - char* hash; - block_off_t v_pos; - struct snapraid_block* block; - char* e; + } else if (strcmp(tag, "blksize") == 0) { + block_off_t blksize; - if (!file) { - fprintf(stderr, "Unexpected 'blk' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - } - - pos = s; - s = strtoken(s); - hash = s; - - ret = stru32(pos, &v_pos); + ret = stru32(s, &blksize); if (ret != 0) { - fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); + fprintf(stderr, "Invalid 'blksize' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } - if (block_index >= file->blockmax) { - fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); + if (blksize != state->block_size) { + fprintf(stderr, "Mismatching 'blksize' and 'block_size' specification in '%s' at line %u\n", path, line); + fprintf(stderr, "Please restore the 'block_size' value in the configuration file to '%u'\n", blksize / 1024); exit(EXIT_FAILURE); } - - block = &file->blockvec[block_index]; - - if (block->parity_pos != POS_INVALID) { - fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - } - - block->parity_pos = v_pos; - - /* set the hash only if present */ - if (*hash != 0) { - e = strdechex(block->hash, HASH_MAX, hash); - if (e) { - fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - } - block->is_hashed = 1; - } - - /* insert the block in the block array */ - tommy_array_grow(&disk->blockarr, block->parity_pos + 1); - tommy_array_set(&disk->blockarr, block->parity_pos, block); - - /* check for termination of the block list */ - ++block_index; - if (block_index == file->blockmax) { - file = 0; - disk = 0; - } - - /* stat */ - ++count_block; } else { fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line); exit(EXIT_FAILURE); @@ -371,6 +385,7 @@ void state_write(struct snapraid_state* state) unsigned i; unsigned count_file; unsigned count_block; + int ret; count_file = 0; count_block = 0; @@ -384,6 +399,12 @@ void state_write(struct snapraid_state* state) exit(EXIT_FAILURE); } + ret = fprintf(f, "blksize %u\n", state->block_size); + if (ret < 0) { + fprintf(stderr, "Error writing the state file '%s' in fprintf(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + /* for each disk */ for(i=0;idiskarr);++i) { tommy_node* j; @@ -393,7 +414,6 @@ void state_write(struct snapraid_state* state) for(j=disk->filelist;j!=0;j=j->next) { block_off_t k; struct snapraid_file* file = j->data; - int ret; ret = fprintf(f,"file %s %lld %ld %s\n", disk->name, file->size, file->mtime, file->sub); if (ret < 0) {