From 3698064040c6f0b913e6a1ca5e7121402a07fad0 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 17 Aug 2013 16:17:53 +0200 Subject: [PATCH] Uses a binary content file. --- HISTORY | 3 +- Makefile.am | 4 +- TODO | 4 - snapraid.c | 7 + state.c | 1478 +++++++++++++++++++++++++++++++++++++++++++++------ state.h | 1 + util.c | 308 ++++++++++- util.h | 90 ++++ 8 files changed, 1704 insertions(+), 191 deletions(-) diff --git a/HISTORY b/HISTORY index 8d5df31c..55284793 100644 --- a/HISTORY +++ b/HISTORY @@ -1,7 +1,7 @@ SnapRAID HISTORY ================ -4.0 2013/7 (BETA) +4.0 2013/8 (BETA) ================= * New 'scrub' command to periodically check the oldest blocks for silent errors without the need to scan the whole array. @@ -9,6 +9,7 @@ SnapRAID HISTORY distribution, and the silent error status of the array. * Added the new Spooky hash. It's faster in 64 bit architectures. To convert you can use the new 'rehash' command. + * Changed to a binary content file to improve speed and reduce size. * Added a hash seed to make harder intentional collision attacks. * When insering files for the first time, sort them by their physical address to improve read performance. diff --git a/Makefile.am b/Makefile.am index 36868206..f90a193b 100644 --- a/Makefile.am +++ b/Makefile.am @@ -170,7 +170,7 @@ endif mkdir test/disk2 $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --force-empty -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check - echo --- Use the with the hole + echo --- Use with the hole $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf check rm -r test/disk1/_a @@ -225,7 +225,7 @@ endif $(TESTENV) ./mktest$(EXEEXT) generate 4 3 $(CHECKCOUNT) $(CHECKSIZE) $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 100 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check -l test.log - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 700 sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-force-content-text -t 700 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1000 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check -l test.log diff --git a/TODO b/TODO index 1e418a1a..bad228ec 100644 --- a/TODO +++ b/TODO @@ -3,10 +3,6 @@ SnapRAID TODO This is the list of TODO items for SnapRAID. -- v4.1 - -* Support names including \r and \n for Mac. See proposed patch from Matthew - - v4.x generic * In the import list, uses also all the blocks in the array. diff --git a/snapraid.c b/snapraid.c index b9cbeaf1..572afed6 100644 --- a/snapraid.c +++ b/snapraid.c @@ -95,6 +95,7 @@ void usage(void) #define OPT_TEST_FORCE_SCRUB 271 #define OPT_TEST_FORCE_SCRUB_EVEN 272 #define OPT_TEST_FORCE_CONTENT_WRITE 273 +#define OPT_TEST_FORCE_CONTENT_TEXT 274 #if HAVE_GETOPT_LONG struct option long_options[] = { @@ -168,6 +169,9 @@ struct option long_options[] = { /* Force write of the content file even if no modification is done. */ { "test-force-content-write", 0, 0, OPT_TEST_FORCE_CONTENT_WRITE }, + /* Force the use of text content file . */ + { "test-force-content-text", 0, 0, OPT_TEST_FORCE_CONTENT_TEXT }, + { 0, 0, 0, 0 } }; #endif @@ -378,6 +382,9 @@ int main(int argc, char* argv[]) case OPT_TEST_FORCE_CONTENT_WRITE : opt.force_content_write = 1; break; + case OPT_TEST_FORCE_CONTENT_TEXT : + opt.force_content_text = 1; + break; default: fprintf(stderr, "Unknown option '%c'\n", (char)c); exit(EXIT_FAILURE); diff --git a/state.c b/state.c index 0020b3ee..6e28a1cb 100644 --- a/state.c +++ b/state.c @@ -821,10 +821,53 @@ static void state_content_check(struct snapraid_state* state, const char* path) } } -void state_read(struct snapraid_state* state) +/** + * Checks if a block position is used by at least one disk. + */ +static int is_block_used(struct snapraid_state* state, block_off_t pos) +{ + tommy_node* i; + + /* check for each disk if block is really used */ + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* disk = i->data; + struct snapraid_block* block = disk_block_get(disk, pos); + + if (block != BLOCK_EMPTY) + return 1; + } + + return 0; +} + +/** + * Checks if a block position in a disk is deleted. + */ +static int is_block_deleted(struct snapraid_disk* disk, block_off_t pos) +{ + struct snapraid_block* block = disk_block_get(disk, pos); + + return block_state_get(block) == BLOCK_STATE_DELETED; +} + +/** + * Finds a disk by name. + */ +static struct snapraid_disk* find_disk(struct snapraid_state* state, const char* name) +{ + tommy_node* i; + + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* disk = i->data; + if (strcmp(disk->name, name) == 0) + return disk; + } + + return 0; +} + +static void state_read_text(struct snapraid_state* state, const char* path, STREAM* f, time_t save_time) { - STREAM* f; - char path[PATH_MAX]; struct snapraid_disk* disk; struct snapraid_file* file; block_off_t blockidx; @@ -835,83 +878,19 @@ void state_read(struct snapraid_state* state) unsigned count_hardlink; unsigned count_symlink; unsigned count_dir; - tommy_node* node; oathash_t hash; - struct stat st; - int ret; - - count_file = 0; - count_block = 0; - count_hardlink = 0; - count_symlink = 0; - count_dir = 0; - hash = 0; - - /* iterate over all the available content files and load the first one present */ - f = 0; - node = tommy_list_head(&state->contentlist); - while (node) { - struct snapraid_content* content = node->data; - pathcpy(path, sizeof(path), content->content); - - if (state->opt.gui) { - fprintf(stdlog, "content:%s\n", path); - fflush(stdlog); - } - printf("Loading state from %s...\n", path); - - f = sopen_read(path); - if (f != 0) { - /* if openend stop the search */ - break; - } else { - /* if it's real error of an existing file, abort */ - if (errno != ENOENT) { - fprintf(stderr, "Error opening the content file '%s'. %s.\n", path, strerror(errno)); - exit(EXIT_FAILURE); - } - - /* otherwise continue */ - if (node->next) { - fprintf(stderr, "warning: Content file '%s' not found, trying with another copy...\n", path); - } - } - - node = node->next; - } - - /* if not found, assume empty */ - if (!f) { - /* create the initial mapping */ - state_map(state); - - fprintf(stderr, "warning: No content file found. Assuming empty.\n"); - return; - } - - /* get the date of the content file */ - ret = fstat(shandle(f), &st); - if (ret != 0) { - fprintf(stderr, "Error stating the content file '%s'. %s.\n", path, strerror(errno)); - exit(EXIT_FAILURE); - } - - /* start with a undefined default. */ - /* it's for compatibility with version 1.0 where MD5 was implicit. */ - state->hash = HASH_UNDEFINED; - - /* start with a zero seed, it was the default in old versions */ - memset(state->hashseed, 0, HASH_SIZE); - - /* previous hash, start with an undefined value */ - state->prevhash = HASH_UNDEFINED; - memset(state->prevhashseed, 0, HASH_SIZE); disk = 0; file = 0; line = 1; blockidx = 0; blockmax = 0; + count_file = 0; + count_block = 0; + count_hardlink = 0; + count_symlink = 0; + count_dir = 0; + hash = 0; while (1) { char buffer[PATH_MAX]; @@ -1020,7 +999,7 @@ void state_read(struct snapraid_state* state) /* set a fake info block, in case of upgrading from an old version */ /* the real block info will overwrite this */ - info = info_make(st.st_mtime, 0, 0); + info = info_make(save_time, 0, 0); /* insert the info in the array */ info_set(&state->infoarr, v_pos, info); @@ -1150,7 +1129,6 @@ void state_read(struct snapraid_state* state) uint64_t v_mtime_sec; uint32_t v_mtime_nsec; uint64_t v_inode; - tommy_node* i; hash = oathash8(hash, 'f'); @@ -1242,12 +1220,8 @@ void state_read(struct snapraid_state* state) hash = oathashs(hash, sub); /* find the disk */ - for(i=state->disklist;i!=0;i=i->next) { - disk = i->data; - if (strcmp(disk->name, buffer) == 0) - break; - } - if (!i) { + disk = find_disk(state, buffer); + if (!disk) { fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); exit(EXIT_FAILURE); } @@ -1273,8 +1247,6 @@ void state_read(struct snapraid_state* state) ++count_file; } else if (strcmp(tag, "hole") == 0) { /* hole */ - tommy_node* i; - hash = oathash8(hash, 'h'); if (file) { @@ -1290,12 +1262,8 @@ void state_read(struct snapraid_state* state) hash = oathashs(hash, buffer); /* find the disk */ - for(i=state->disklist;i!=0;i=i->next) { - disk = i->data; - if (strcmp(disk->name, buffer) == 0) - break; - } - if (!i) { + disk = find_disk(state, buffer); + if (!disk) { fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); exit(EXIT_FAILURE); } @@ -1304,7 +1272,6 @@ void state_read(struct snapraid_state* state) char sub[PATH_MAX]; char linkto[PATH_MAX]; char tokento[32]; - tommy_node* i; struct snapraid_link* link; hash = oathash8(hash, 's'); @@ -1365,12 +1332,8 @@ void state_read(struct snapraid_state* state) } /* find the disk */ - for(i=state->disklist;i!=0;i=i->next) { - disk = i->data; - if (strcmp(disk->name, buffer) == 0) - break; - } - if (!i) { + disk = find_disk(state, buffer); + if (!disk) { fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); exit(EXIT_FAILURE); } @@ -1389,7 +1352,6 @@ void state_read(struct snapraid_state* state) char sub[PATH_MAX]; char linkto[PATH_MAX]; char tokento[32]; - tommy_node* i; struct snapraid_link* link; hash = oathash8(hash, 'a'); @@ -1450,12 +1412,8 @@ void state_read(struct snapraid_state* state) } /* find the disk */ - for(i=state->disklist;i!=0;i=i->next) { - disk = i->data; - if (strcmp(disk->name, buffer) == 0) - break; - } - if (!i) { + disk = find_disk(state, buffer); + if (!disk) { fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); exit(EXIT_FAILURE); } @@ -1472,7 +1430,6 @@ void state_read(struct snapraid_state* state) } else if (strcmp(tag, "dir") == 0) { /* dir */ char sub[PATH_MAX]; - tommy_node* i; struct snapraid_dir* dir; hash = oathash8(hash, 'r'); @@ -1503,12 +1460,8 @@ void state_read(struct snapraid_state* state) hash = oathashs(hash, sub); /* find the disk */ - for(i=state->disklist;i!=0;i=i->next) { - disk = i->data; - if (strcmp(disk->name, buffer) == 0) - break; - } - if (!i) { + disk = find_disk(state, buffer); + if (!disk) { fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); exit(EXIT_FAILURE); } @@ -1661,7 +1614,7 @@ void state_read(struct snapraid_state* state) if (sign != hash) { fprintf(stderr, "Mismatching 'sign' in '%s' at line %u\n", path, line); if (!state->opt.skip_sign) { - fprintf(stderr, "Likely this content file is damaged. Use an alternate copy\n"); + fprintf(stderr, "This content file is damaged! Use an alternate copy.\n"); exit(EXIT_FAILURE); } } @@ -1701,23 +1654,13 @@ void state_read(struct snapraid_state* state) exit(EXIT_FAILURE); } - sclose(f); - - if (state->hash == HASH_UNDEFINED) { - fprintf(stderr, "The checksum to use is not specified.\n"); - fprintf(stderr, "This happens because you are likely upgrading from SnapRAID 1.0.\n"); - fprintf(stderr, "To use a new SnapRAID you must restart from scratch,\n"); - fprintf(stderr, "deleting all the content and parity files.\n"); + /* check that the stored parity size matches the loaded state */ + if (blockmax != parity_size(state)) { + fprintf(stderr, "Internal incosistency in parity size in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } - state->loaded_blockmax = blockmax; - /* update the mapping */ - state_map(state); - - state_content_check(state, path); - if (state->opt.verbose) { printf("\tfile %u\n", count_file); printf("\tblock %u\n", count_block); @@ -1727,19 +1670,16 @@ void state_read(struct snapraid_state* state) } } -void state_write(struct snapraid_state* state) +static void state_write_text(struct snapraid_state* state, STREAM* f) { - STREAM* f; unsigned count_file; unsigned count_block; unsigned count_hardlink; unsigned count_symlink; unsigned count_dir; - unsigned count_content; tommy_node* i; block_off_t b; block_off_t blockmax; - unsigned k; oathash_t hash; count_file = 0; @@ -1752,37 +1692,6 @@ void state_write(struct snapraid_state* state) /* blocks of all array */ blockmax = parity_size(state); - /* count the content files */ - count_content = 0; - i = tommy_list_head(&state->contentlist); - while (i) { - struct snapraid_content* content = i->data; - printf("Saving state to %s...\n", content->content); - ++count_content; - i = i->next; - } - - /* open all the content files */ - f = sopen_multi_write(count_content); - if (!f) { - fprintf(stderr, "Error opening the content files.\n"); - exit(EXIT_FAILURE); - } - - k = 0; - i = tommy_list_head(&state->contentlist); - while (i) { - struct snapraid_content* content = i->data; - char tmp[PATH_MAX]; - pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); - if (sopen_multi_file(f, k, tmp) != 0) { - fprintf(stderr, "Error opening the content file '%s'. %s.\n", tmp, strerror(errno)); - exit(EXIT_FAILURE); - } - ++k; - i = i->next; - } - sputsl("blksize ", f); hash = oathash8(hash, 'z'); sputu32(state->block_size, f); @@ -2131,6 +2040,1249 @@ void state_write(struct snapraid_state* state) exit(EXIT_FAILURE); } + if (state->opt.verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + printf("\thardlink %u\n", count_hardlink); + printf("\tsymlink %u\n", count_symlink); + printf("\temptydir %u\n", count_dir); + } +} + +/** + * Flush the file checking the final CRC. + * We exploit the fact that the CRC is always stored in the last 4 bytes. + */ +static void decoding_error(const char* path, STREAM* f) +{ + unsigned char buf[4]; + uint32_t crc_stored; + uint32_t crc_computed; + + if (seof(f)) { + fprintf(stderr, "Unexpected end of content file '%s' at offset %"PRIi64"\n", path, stell(f)); + fprintf(stderr, "This content file is truncated. Use an alternate copy.\n"); + exit(EXIT_FAILURE); + } + + if (serror(f)) { + fprintf(stderr, "Error reading the content file '%s' at offset %"PRIi64"\n", path, stell(f)); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "Decoding error in '%s' at offset %"PRIi64"\n", path, stell(f)); + + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + + /* read until the end of the file */ + while (1) { + int c = sgetc(f); + if (c == EOF) { + break; + } + + /* keep the last four bytes */ + buf[0] = buf[1]; + buf[1] = buf[2]; + buf[2] = buf[3]; + buf[3] = c; + } + + if (serror(f)) { + fprintf(stderr, "Error flushing the content file '%s' at offset %"PRIi64"\n", path, stell(f)); + exit(EXIT_FAILURE); + } + + /* get the stored crc from the last four bytes */ + crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24; + + /* get the computed crc */ + crc_computed = scrc(f); + + /* adjust the stored crc to include itself */ + crc_stored = crc32c(crc_stored, buf, 4); + + if (crc_computed != crc_stored) { + fprintf(stderr, "Mismatching CRC in '%s'\n", path); + fprintf(stderr, "This content file is damaged! Use an alternate copy.\n"); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "The file CRC is correct! This seems a SnapRAID bug!\n"); + } +} + +static void state_read_binary(struct snapraid_state* state, const char* path, STREAM* f) +{ + block_off_t blockmax; + unsigned count_file; + unsigned count_block; + unsigned count_hardlink; + unsigned count_symlink; + unsigned count_dir; + int crc_checked; + char buffer[PATH_MAX]; + int ret; + + blockmax = 0; + count_file = 0; + count_block = 0; + count_hardlink = 0; + count_symlink = 0; + count_dir = 0; + crc_checked = 0; + + ret = sread(f, buffer, 10); + if (ret < 0 || memcmp(buffer, "SnapRAID\n\3", 10) != 0) { + decoding_error(path, f); + fprintf(stderr, "Invalid header!\n"); + exit(EXIT_FAILURE); + } + + while (1) { + char buffer[PATH_MAX]; + int c; + + /* read the command */ + c = sgetc(f); + if (c == EOF) { + break; + } + + if (c == 'f') { + /* file */ + char sub[PATH_MAX]; + uint64_t v_size; + uint64_t v_mtime_sec; + uint32_t v_mtime_nsec; + uint64_t v_inode; + uint32_t v_idx; + struct snapraid_file* file; + struct snapraid_disk* disk; + + ret = sgetbs(f, buffer, sizeof(buffer)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetb64(f, &v_size); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetb64(f, &v_mtime_sec); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &v_mtime_nsec); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* ignore nanoseconds if asked to find by name */ + if (state->opt.force_by_name) + v_mtime_nsec = FILE_MTIME_NSEC_INVALID; + + ret = sgetb64(f, &v_inode); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetbs(f, sub, sizeof(sub)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + if (!*sub) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* find the disk */ + disk = find_disk(state, buffer); + if (!disk) { + decoding_error(path, f); + fprintf(stderr, "Disk named '%s' not found!\n", buffer); + exit(EXIT_FAILURE); + } + + /* allocate the file */ + file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0); + + /* insert the file in the file containers */ + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); + + /* reads all the blocks */ + v_idx = 0; + while (v_idx < file->blockmax) { + block_off_t v_pos; + uint32_t v_count; + + /* get the "blk"/"new"/"chg" command */ + c = sgetc(f); + + ret = sgetb32(f, &v_pos); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &v_count); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (v_idx + v_count > file->blockmax) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency in block number!\n"); + exit(EXIT_FAILURE); + } + + if (v_pos + v_count > blockmax) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency in block size!\n"); + exit(EXIT_FAILURE); + } + + /* grow the array */ + tommy_array_grow(&disk->blockarr, v_pos + v_count); + + /* fill the blocks in the run */ + while (v_count) { + struct snapraid_block* block = &file->blockvec[v_idx]; + + if (block->parity_pos != POS_INVALID) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency in block position!\n"); + exit(EXIT_FAILURE); + } + + switch (c) { + case 'b' : + block_state_set(block, BLOCK_STATE_BLK); + break; + case 'n' : + block_state_set(block, BLOCK_STATE_NEW); + break; + case 'g' : + block_state_set(block, BLOCK_STATE_CHG); + break; + default: + decoding_error(path, f); + fprintf(stderr, "Invalid block type!\n"); + exit(EXIT_FAILURE); + } + + block->parity_pos = v_pos; + + /* read the hash only for 'blk/chg', and not for 'new' */ + if (c != 'n') { + /* set the hash only if present */ + ret = sread(f, block->hash, HASH_SIZE); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + } + + /* we must not overwrite existing blocks */ + if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency in block existence!\n"); + exit(EXIT_FAILURE); + } + + /* insert the block in the block array */ + tommy_array_set(&disk->blockarr, v_pos, block); + + /* stat */ + ++count_block; + + /* go to the next block */ + ++v_idx; + ++v_pos; + --v_count; + } + } + + /* stat */ + ++count_file; + } else if (c == 'i') { + /* "inf" command */ + snapraid_info info; + uint32_t v_pos; + uint32_t v_oldest; + + ret = sgetb32(f, &v_oldest); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &blockmax); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + v_pos = 0; + while (v_pos < blockmax) { + int rehash; + int bad; + uint32_t t; + uint32_t flag; + uint32_t v_count; + + ret = sgetb32(f, &v_count); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (v_pos + v_count > blockmax) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency in info size!\n"); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &flag); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* if there is an info */ + if ((flag & 1) != 0) { + /* read the time */ + ret = sgetb32(f, &t); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* analyze the flags */ + bad = (flag & 2) != 0; + rehash = (flag & 4) != 0; + + if (rehash && state->prevhash == HASH_UNDEFINED) { + decoding_error(path, f); + fprintf(stderr, "Internal incosistency for missing previous checksum!\n"); + exit(EXIT_FAILURE); + } + + info = info_make(t + v_oldest, bad, rehash); + } else { + info = 0; + } + + while (v_count) { + /* insert the info in the array */ + info_set(&state->infoarr, v_pos, info); + + /* go to next block */ + ++v_pos; + --v_count; + } + } + } else if (c == 'h') { + /* hole */ + uint32_t v_pos; + block_off_t blockdiskmax; + struct snapraid_disk* disk; + + ret = sgetbs(f, buffer, sizeof(buffer)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* find the disk */ + disk = find_disk(state, buffer); + if (!disk) { + decoding_error(path, f); + fprintf(stderr, "Disk named '%s' not found!\n", buffer); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &blockdiskmax); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (blockdiskmax > blockmax) { + fprintf(stderr, "Internal inconsistency in disk size!\n"); + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* grow the array */ + tommy_array_grow(&disk->blockarr, blockdiskmax); + + v_pos = 0; + while (v_pos < blockdiskmax) { + uint32_t flag; + uint32_t v_count; + + ret = sgetb32(f, &v_count); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (v_pos + v_count > blockdiskmax) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency in hole size!\n"); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &flag); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* if it's a run of deleted blocks */ + if ((flag & 1) != 0) { + while (v_count) { + struct snapraid_deleted* deleted; + + /* allocate a new deleted block */ + deleted = deleted_alloc(); + + /* insert it in the list of deleted blocks */ + tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); + + /* set the position */ + deleted->block.parity_pos = v_pos; + + /* read the hash */ + ret = sread(f, deleted->block.hash, HASH_SIZE); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* we must not overwrite existing blocks */ + if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + decoding_error(path, f); + fprintf(stderr, "Internal inconsistency for used hole!\n"); + exit(EXIT_FAILURE); + } + + /* insert the block in the block array */ + tommy_array_set(&disk->blockarr, v_pos, &deleted->block); + + /* go to next block */ + ++v_pos; + --v_count; + } + } else { + /* go to the next run */ + v_pos += v_count; + } + } + } else if (c == 's') { + /* symlink */ + char sub[PATH_MAX]; + char linkto[PATH_MAX]; + struct snapraid_link* link; + struct snapraid_disk* disk; + + ret = sgetbs(f, buffer, sizeof(buffer)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetbs(f, sub, sizeof(sub)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetbs(f, linkto, sizeof(linkto)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (!*sub || !*linkto) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* find the disk */ + disk = find_disk(state, buffer); + if (!disk) { + decoding_error(path, f); + fprintf(stderr, "Disk named '%s' not found!\n", buffer); + exit(EXIT_FAILURE); + } + + /* allocate the link as symbolic link */ + link = link_alloc(sub, linkto, FILE_IS_SYMLINK); + + /* insert the link in the link containers */ + tommy_hashdyn_insert(&disk->linkset, &link->nodeset, link, link_name_hash(link->sub)); + tommy_list_insert_tail(&disk->linklist, &link->nodelist, link); + + /* stat */ + ++count_symlink; + } else if (c == 'a') { + /* hardlink */ + char sub[PATH_MAX]; + char linkto[PATH_MAX]; + struct snapraid_link* link; + struct snapraid_disk* disk; + + ret = sgetbs(f, buffer, sizeof(buffer)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetbs(f, sub, sizeof(sub)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetbs(f, linkto, sizeof(linkto)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (!*sub || !*linkto) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* find the disk */ + disk = find_disk(state, buffer); + if (!disk) { + decoding_error(path, f); + fprintf(stderr, "Disk named '%s' not found!\n", buffer); + exit(EXIT_FAILURE); + } + + /* allocate the link as hard link */ + link = link_alloc(sub, linkto, FILE_IS_HARDLINK); + + /* insert the link in the link containers */ + tommy_hashdyn_insert(&disk->linkset, &link->nodeset, link, link_name_hash(link->sub)); + tommy_list_insert_tail(&disk->linklist, &link->nodelist, link); + + /* stat */ + ++count_hardlink; + } else if (c == 'r') { + /* dir */ + char sub[PATH_MAX]; + struct snapraid_dir* dir; + struct snapraid_disk* disk; + + ret = sgetbs(f, buffer, sizeof(buffer)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetbs(f, sub, sizeof(sub)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (!*sub) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* find the disk */ + disk = find_disk(state, buffer); + if (!disk) { + decoding_error(path, f); + fprintf(stderr, "Disk named '%s' not found!\n", buffer); + exit(EXIT_FAILURE); + } + + /* allocate the dir */ + dir = dir_alloc(sub); + + /* insert the dir in the dir containers */ + tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub)); + tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir); + + /* stat */ + ++count_dir; + } else if (c == 'c') { + c = sgetc(f); + switch (c) { + case 'u' : + state->hash = HASH_MURMUR3; + break; + case 'k' : + state->hash = HASH_SPOOKY2; + break; + default: + decoding_error(path, f); + fprintf(stderr, "Invalid checksum!\n"); + exit(EXIT_FAILURE); + } + + /* read the seed */ + ret = sread(f, state->hashseed, HASH_SIZE); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + } else if (c == 'C') { + c = sgetc(f); + switch (c) { + case 'u' : + state->prevhash = HASH_MURMUR3; + break; + case 'k' : + state->prevhash = HASH_SPOOKY2; + break; + default: + decoding_error(path, f); + fprintf(stderr, "Invalid checksum!\n"); + exit(EXIT_FAILURE); + } + + /* read the seed */ + ret = sread(f, state->prevhashseed, HASH_SIZE); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + } else if (c == 'z') { + block_off_t blksize; + + ret = sgetb32(f, &blksize); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + if (blksize != state->block_size) { + decoding_error(path, f); + fprintf(stderr, "Mismatching 'blksize' and 'block_size' specification!\n"); + fprintf(stderr, "Please restore the 'block_size' value in the configuration file to '%u'\n", blksize / 1024); + exit(EXIT_FAILURE); + } + } else if (c == 'm') { + struct snapraid_map* map; + char uuid[UUID_MAX]; + uint32_t v_pos; + + ret = sgetbs(f, buffer, sizeof(buffer)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + ret = sgetb32(f, &v_pos); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + /* read the uuid */ + ret = sgetbs(f, uuid, sizeof(uuid)); + if (ret < 0) { + decoding_error(path, f); + exit(EXIT_FAILURE); + } + + map = map_alloc(buffer, v_pos, uuid); + + tommy_list_insert_tail(&state->maplist, &map->node, map); + } else if (c == 'N') { + uint32_t crc_stored; + uint32_t crc_computed; + + /* get the crc before reading it from the file */ + crc_computed = scrc(f); + + ret = sgetble32(f, &crc_stored); + if (ret < 0) { + /* here don't call decoding_error() because it's too late to get the crc */ + fprintf(stderr, "Error reading the CRC in '%s' at offset %"PRIi64"\n", path, stell(f)); + fprintf(stderr, "This content file is damaged! Use an alternate copy.\n"); + exit(EXIT_FAILURE); + } + + if (crc_stored != crc_computed) { + /* here don't call decoding_error() because it's too late to get the crc */ + fprintf(stderr, "Mismatching CRC in '%s'\n", path); + fprintf(stderr, "This content file is damaged! Use an alternate copy.\n"); + exit(EXIT_FAILURE); + } + + crc_checked = 1; + } else { + decoding_error(path, f); + fprintf(stderr, "Invalid command '%c'!\n", (char)c); + exit(EXIT_FAILURE); + } + } + + if (serror(f)) { + fprintf(stderr, "Error reading the content file '%s' at offset %"PRIi64"\n", path, stell(f)); + exit(EXIT_FAILURE); + } + + if (!crc_checked) { + fprintf(stderr, "Finished reading '%s' without finding the CRC\n", path); + fprintf(stderr, "This content file is truncated or damaged! Use an alternate copy.\n"); + exit(EXIT_FAILURE); + } + + /* check that the stored parity size matches the loaded state */ + if (blockmax != parity_size(state)) { + fprintf(stderr, "Internal incosistency in parity size in '%s' at offset %"PRIi64"\n", path, stell(f)); + exit(EXIT_FAILURE); + } + state->loaded_blockmax = blockmax; + + if (state->opt.verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + printf("\thardlink %u\n", count_hardlink); + printf("\tsymlink %u\n", count_symlink); + printf("\temptydir %u\n", count_dir); + } +} + +static void state_write_binary(struct snapraid_state* state, STREAM* f) +{ + unsigned count_file; + unsigned count_block; + unsigned count_hardlink; + unsigned count_symlink; + unsigned count_dir; + tommy_node* i; + block_off_t b; + block_off_t blockmax; + block_off_t begin; + time_t info_oldest; + int info_has_rehash; + + count_file = 0; + count_block = 0; + count_hardlink = 0; + count_symlink = 0; + count_dir = 0; + + /* blocks of all array */ + blockmax = parity_size(state); + + /* write header */ + swrite("SnapRAID\n\3", 10, f); + + /* clear the info for unused blocks */ + /* and get some other info */ + info_oldest = 0; /* oldest time in info */ + info_has_rehash = 0; /* if there is a rehash info */ + for(b=0;binfoarr, b); + + /* only if there is some info to store */ + if (info) { + time_t time = info_get_time(info); + + if (!info_oldest || time < info_oldest) + info_oldest = time; + + if (info_get_rehash(info)) + info_has_rehash = 1; + } + } else { + /* clear any previous info */ + info_set(&state->infoarr, b, 0); + } + } + + sputc('z', f); + sputb32(state->block_size, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + sputc('c', f); + if (state->hash == HASH_MURMUR3) { + sputc('u', f); + } else if (state->hash == HASH_SPOOKY2) { + sputc('k', f); + } else { + fprintf(stderr, "Unexpected hash when writing the content file '%s'.\n", serrorfile(f)); + exit(EXIT_FAILURE); + } + swrite(state->hashseed, HASH_SIZE, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + /* previous hash only present */ + if (state->prevhash != HASH_UNDEFINED) { + /* if at least one rehash tag found, we have to save the previous hash */ + if (info_has_rehash) { + sputc('C', f); + if (state->prevhash == HASH_MURMUR3) { + sputc('u', f); + } else if (state->hash == HASH_SPOOKY2) { + sputc('k', f); + } else { + fprintf(stderr, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); + exit(EXIT_FAILURE); + } + swrite(state->prevhashseed, HASH_SIZE, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + } + } + + /* for each map */ + for(i=state->maplist;i!=0;i=i->next) { + struct snapraid_map* map = i->data; + sputc('m', f); + sputbs(map->name, f); + sputb32(map->position, f); + sputbs(map->uuid, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + } + + /* save the info */ + sputc('i', f); + + sputb32(info_oldest, f); + + /* read the blockmax, this is also going to be used in a lot of checks */ + sputb32(blockmax, f); + + /* write the info for each block */ + begin = 0; + while (begin < blockmax) { + snapraid_info info; + block_off_t end; + time_t t; + unsigned flag; + + info = info_get(&state->infoarr, begin); + + /* find the end of run of blocks */ + end = begin + 1; + while (end < blockmax + && info == info_get(&state->infoarr, end)) + { + ++end; + } + + sputb32(end - begin, f); + + /* if there is info */ + if (info) { + /* other flags */ + flag = 1; /* time is present */ + if (info_get_bad(info)) + flag |= 2; + if (info_get_rehash(info)) + flag |= 4; + sputb32(flag, f); + + t = info_get_time(info) - info_oldest; + sputb32(t, f); + } else { + /* write a special 0 flag to mark missing info */ + sputb32(0, f); + } + + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + /* next begin position */ + begin = end; + } + + /* for each disk */ + for(i=state->disklist;i!=0;i=i->next) { + tommy_node* j; + struct snapraid_disk* disk = i->data; + block_off_t blockdiskmax; + + /* for each file */ + for(j=disk->filelist;j!=0;j=j->next) { + struct snapraid_file* file = j->data; + struct snapraid_block* blockvec = file->blockvec; + uint64_t size; + uint64_t mtime_sec; + int32_t mtime_nsec; + uint64_t inode; + + size = file->size; + mtime_sec = file->mtime_sec; + mtime_nsec = file->mtime_nsec; + inode = file->inode; + + sputc('f', f); + sputbs(disk->name, f); + sputb64(size, f); + sputb64(mtime_sec, f); + sputb32(mtime_nsec, f); + sputb64(inode, f); + sputbs(file->sub, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + /* for all the blocks of the file */ + begin = 0; + while (begin < file->blockmax) { + unsigned block_state = block_state_get(blockvec + begin); + uint32_t v_pos = blockvec[begin].parity_pos; + uint32_t v_count; + + block_off_t end; + + /* find the end of run of blocks */ + end = begin + 1; + while (end < file->blockmax + && block_state == block_state_get(blockvec + end) + && blockvec[end-1].parity_pos + 1 == blockvec[end].parity_pos) + { + ++end; + } + + switch (block_state) { + case BLOCK_STATE_BLK : + sputc('b', f); + break; + case BLOCK_STATE_NEW : + sputc('n', f); + break; + case BLOCK_STATE_CHG : + sputc('g', f); + break; + default: + fprintf(stderr, "Internal state inconsistency in saving for block %u state %u\n", v_pos, block_state); + exit(EXIT_FAILURE); + } + + sputb32(v_pos, f); + + v_count = end - begin; + sputb32(v_count, f); + + if (block_state != BLOCK_STATE_NEW) { + for(b=begin;blinklist;j!=0;j=j->next) { + struct snapraid_link* link = j->data; + + switch (link_flag_get(link, FILE_IS_LINK_MASK)) { + case FILE_IS_HARDLINK : + sputc('a', f); + ++count_hardlink; + break; + case FILE_IS_SYMLINK : + sputc('s', f); + ++count_symlink; + break; + } + + sputbs(disk->name, f); + sputbs(link->sub, f); + sputbs(link->linkto, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + } + + /* for each dir */ + for(j=disk->dirlist;j!=0;j=j->next) { + struct snapraid_dir* dir = j->data; + + sputc('r', f); + sputbs(disk->name, f); + sputbs(dir->sub, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + ++count_dir; + } + + /* maximum block for this disk */ + blockdiskmax = tommy_array_size(&disk->blockarr); + + /* skip blocks over the end of the parity */ + if (blockdiskmax > blockmax) + blockdiskmax = blockmax; + + /* skip empty blocks at the end of the disk */ + while (blockdiskmax > 0 && disk_block_get(disk, blockdiskmax - 1) == BLOCK_EMPTY) + --blockdiskmax; + + /* write the hole list only if the disk is not completely empty */ + /* this is required to write no reference to a removed disk */ + if (blockdiskmax) { + /* write the list of deleted blocks */ + sputc('h', f); + sputbs(disk->name, f); + sputb32(blockdiskmax, f); + + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + /* write the deleted info for each block */ + begin = 0; + while (begin < blockdiskmax) { + int is_deleted; + block_off_t end; + + is_deleted = is_block_deleted(disk, begin); + + /* find the end of run of blocks */ + end = begin + 1; + while (end < blockdiskmax + && is_deleted == is_block_deleted(disk, end)) + { + ++end; + } + + sputb32(end - begin, f); + + if (is_deleted) { + /* write the run of deleted blocks with hash */ + sputb32(1, f); + + /* write all the hash */ + while (begin < end) { + struct snapraid_block* block = disk_block_get(disk, begin); + + swrite(block->hash, HASH_SIZE, f); + + ++begin; + } + } else { + /* write the run of blocks without hash */ + /* they can be either used or empty blocks */ + sputb32(0, f); + + /* next begin position */ + begin = end; + } + + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + } + } + } + + sputc('N', f); + sputble32(scrc(f), f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + if (state->opt.verbose) { + printf("\tfile %u\n", count_file); + printf("\tblock %u\n", count_block); + printf("\thardlink %u\n", count_hardlink); + printf("\tsymlink %u\n", count_symlink); + printf("\temptydir %u\n", count_dir); + } +} + +void state_read(struct snapraid_state* state) +{ + STREAM* f; + char path[PATH_MAX]; + struct stat st; + tommy_node* node; + int ret; + int c; + + /* iterate over all the available content files and load the first one present */ + f = 0; + node = tommy_list_head(&state->contentlist); + while (node) { + struct snapraid_content* content = node->data; + pathcpy(path, sizeof(path), content->content); + + if (state->opt.gui) { + fprintf(stdlog, "content:%s\n", path); + fflush(stdlog); + } + printf("Loading state from %s...\n", path); + + f = sopen_read(path); + if (f != 0) { + /* if openend stop the search */ + break; + } else { + /* if it's real error of an existing file, abort */ + if (errno != ENOENT) { + fprintf(stderr, "Error opening the content file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + /* otherwise continue */ + if (node->next) { + fprintf(stderr, "warning: Content file '%s' not found, trying with another copy...\n", path); + } + } + + node = node->next; + } + + /* if not found, assume empty */ + if (!f) { + /* create the initial mapping */ + state_map(state); + + fprintf(stderr, "warning: No content file found. Assuming empty.\n"); + return; + } + + /* get the date of the content file */ + ret = fstat(shandle(f), &st); + if (ret != 0) { + fprintf(stderr, "Error stating the content file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + + /* start with a undefined default. */ + /* it's for compatibility with version 1.0 where MD5 was implicit. */ + state->hash = HASH_UNDEFINED; + + /* start with a zero seed, it was the default in old versions */ + memset(state->hashseed, 0, HASH_SIZE); + + /* previous hash, start with an undefined value */ + state->prevhash = HASH_UNDEFINED; + memset(state->prevhashseed, 0, HASH_SIZE); + + /* get the first char to detect the file type */ + c = sgetc(f); + sungetc(c, f); + + if (c == 'S') { + state_read_binary(state, path, f); + } else { + state_read_text(state, path, f, st.st_mtime); + } + + sclose(f); + + if (state->hash == HASH_UNDEFINED) { + fprintf(stderr, "The checksum to use is not specified.\n"); + fprintf(stderr, "This happens because you are likely upgrading from SnapRAID 1.0.\n"); + fprintf(stderr, "To use a new SnapRAID you must restart from scratch,\n"); + fprintf(stderr, "deleting all the content and parity files.\n"); + exit(EXIT_FAILURE); + } + + /* update the mapping */ + state_map(state); + + state_content_check(state, path); +} + +void state_write(struct snapraid_state* state) +{ + STREAM* f; + unsigned count_content; + tommy_node* i; + unsigned k; + + /* count the content files */ + count_content = 0; + i = tommy_list_head(&state->contentlist); + while (i) { + struct snapraid_content* content = i->data; + printf("Saving state to %s...\n", content->content); + ++count_content; + i = i->next; + } + + /* open all the content files */ + f = sopen_multi_write(count_content); + if (!f) { + fprintf(stderr, "Error opening the content files.\n"); + exit(EXIT_FAILURE); + } + + k = 0; + i = tommy_list_head(&state->contentlist); + while (i) { + struct snapraid_content* content = i->data; + char tmp[PATH_MAX]; + pathprint(tmp, sizeof(tmp), "%s.tmp", content->content); + if (sopen_multi_file(f, k, tmp) != 0) { + fprintf(stderr, "Error opening the content file '%s'. %s.\n", tmp, strerror(errno)); + exit(EXIT_FAILURE); + } + ++k; + i = i->next; + } + + if (state->opt.force_content_text) + state_write_text(state, f); + else + state_write_binary(state, f); + /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */ /* than even in a system crash event we have one valid copy of the file. */ @@ -2163,14 +3315,6 @@ void state_write(struct snapraid_state* state) i = i->next; } - if (state->opt.verbose) { - printf("\tfile %u\n", count_file); - printf("\tblock %u\n", count_block); - printf("\thardlink %u\n", count_hardlink); - printf("\tsymlink %u\n", count_symlink); - printf("\temptydir %u\n", count_dir); - } - state->need_write = 0; /* no write needed anymore */ } diff --git a/state.h b/state.h index 02ef9e2b..ad9095bc 100644 --- a/state.h +++ b/state.h @@ -58,6 +58,7 @@ struct snapraid_option { unsigned force_scrub; /**< Force scrub for the specified number of blocks. */ int force_scrub_even; /**< Force scrub of all the even blocks. */ int force_content_write; /**< Force the update of the content file. */ + int force_content_text; /**< Force the use of text version of content file. */ }; struct snapraid_state { diff --git a/util.c b/util.c index 583d9d15..748fb878 100644 --- a/util.c +++ b/util.c @@ -81,6 +81,10 @@ STREAM* sopen_read(const char* file) s->end = s->buffer; s->state = STREAM_STATE_READ; s->state_index = 0; + s->offset = 0; + s->offset_uncached = 0; + s->crc = 0; + s->crc_uncached = 0; return s; } @@ -102,6 +106,10 @@ STREAM* sopen_multi_write(unsigned count) s->end = s->buffer + STREAM_SIZE; s->state = STREAM_STATE_WRITE; s->state_index = 0; + s->offset = 0; + s->offset_uncached = 0; + s->crc = 0; + s->crc_uncached = 0; return s; } @@ -134,22 +142,6 @@ int sopen_multi_file(STREAM* s, unsigned i, const char* file) return 0; } -STREAM* sopen_write(const char* file) -{ - STREAM* s; - - s = sopen_multi_write(1); - if (!s) - return 0; - - if (sopen_multi_file(s, 0, file) != 0) { - sclose(s); - return 0; - } - - return s; -} - int sclose(STREAM* s) { int fail = 0; @@ -201,6 +193,14 @@ int sfill(STREAM* s) return EOF; } + /* update the crc */ + s->crc_uncached = s->crc; + s->crc = crc32c(s->crc, s->buffer, ret); + + /* update the offset */ + s->offset_uncached = s->offset; + s->offset += ret; + s->pos = s->buffer; s->end = s->buffer + ret; @@ -220,6 +220,10 @@ int sflush(STREAM* s) if (!size) return 0; + /* update the crc */ + s->crc = crc32c(s->crc, s->buffer, size); + s->crc_uncached = s->crc; + for(i=0;ihandle_size;++i) { ret = write(s->handle[i].f, s->buffer, size); @@ -230,11 +234,25 @@ int sflush(STREAM* s) } } + /* update the offset */ + s->offset += size; + s->offset_uncached = s->offset; + s->pos = s->buffer; return 0; } +int64_t stell(STREAM* s) +{ + return s->offset_uncached + (s->pos - s->buffer); +} + +uint32_t scrc(STREAM*s) +{ + return crc32c(s->crc_uncached, s->buffer, s->pos - s->buffer); +} + int sgettok(STREAM* f, char* str, int size) { char* i = str; @@ -269,6 +287,35 @@ int sgettok(STREAM* f, char* str, int size) return i - str; } +int sread(STREAM* f, void* void_data, unsigned size) +{ + unsigned char* data = void_data; + + /* if there is enough space in memory */ + if (sptrlookup(f, size)) { + /* optimized version with all the data in memory */ + unsigned char* pos = sptrget(f); + + /* copy it */ + while (size--) { + *data++ = *pos++; + } + + sptrset(f, pos); + } else { + /* standard version using sputc() */ + while (size--) { + int c = sgetc(f); + if (c == EOF) + return -1; + + *data++ = c; + } + } + + return 0; +} + int sgetline(STREAM* f, char* str, int size) { char* i = str; @@ -456,6 +503,89 @@ int sgethex(STREAM* f, void* void_data, int size) return 0; } +int sgetb32(STREAM* f, uint32_t* value) +{ + uint32_t v; + unsigned char b; + unsigned char s; + int c; + + v = 0; + s = 0; +loop: + c = sgetc(f); + if (c == EOF) + return -1; + + b = (unsigned char)c; + if ((b & 0x80) == 0) { + v |= (uint32_t)b << s; + s += 7; + goto loop; + } + + v |= (uint32_t)(b & 0x7f) << s; + + *value = v; + + return 0; +} + +int sgetb64(STREAM* f, uint64_t* value) +{ + uint64_t v; + unsigned char b; + unsigned char s; + int c; + + v = 0; + s = 0; +loop: + c = sgetc(f); + if (c == EOF) + return -1; + + b = (unsigned char)c; + if ((b & 0x80) == 0) { + v |= (uint64_t)b << s; + s += 7; + goto loop; + } + + v |= (uint64_t)(b & 0x7f) << s; + + *value = v; + + return 0; +} + +int sgetble32(STREAM* f, uint32_t* value) +{ + unsigned char buf[4]; + + if (sread(f, buf, 4) != 0) + return -1; + + *value = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24; + + return 0; +} + +int sgetbs(STREAM* f, char* str, int size) +{ + uint32_t len; + + if (sgetb32(f, &len) < 0) + return -1; + + if (len + 1 > (uint32_t)size) + return -1; + + str[len] = 0; + + return sread(f, str, (int)len); +} + int sputs(const char* str, STREAM* f) { while (*str) { @@ -574,6 +704,70 @@ int sputhex(const void* void_data, int size, STREAM* f) return 0; } +int sputb32(uint32_t value, STREAM* s) +{ + unsigned char b; + unsigned char buf[16]; + unsigned i; + + i = 0; +loop: + b = value & 0x7f; + value >>= 7; + + if (value) { + buf[i++] = b; + goto loop; + } + + buf[i++] = b | 0x80; + + return swrite(buf, i, s); +} + +int sputb64(uint64_t value, STREAM* s) +{ + unsigned char b; + unsigned char buf[16]; + unsigned i; + + i = 0; +loop: + b = value & 0x7f; + value >>= 7; + + if (value) { + buf[i++] = b; + goto loop; + } + + buf[i++] = b | 0x80; + + return swrite(buf, i, s); +} + +int sputble32(uint32_t value, STREAM* s) +{ + unsigned char buf[4]; + + buf[0] = value & 0xFF; + buf[1] = (value >> 8) & 0xFF; + buf[2] = (value >> 16) & 0xFF; + buf[3] = (value >> 24) & 0xFF; + + return swrite(buf, 4, s); +} + +int sputbs(const char* str, STREAM* f) +{ + size_t len = strlen(str); + + if (sputb32(len, f) != 0) + return -1; + + return swrite(str, len, f); +} + #if HAVE_FSYNC int ssync(STREAM* s) { @@ -591,6 +785,86 @@ int ssync(STREAM* s) } #endif +/****************************************************************************/ +/* crc */ + +static uint32_t CRC32C[256] = { + 0x00000000, 0xf26b8303, 0xe13b70f7, 0x1350f3f4, + 0xc79a971f, 0x35f1141c, 0x26a1e7e8, 0xd4ca64eb, + 0x8ad958cf, 0x78b2dbcc, 0x6be22838, 0x9989ab3b, + 0x4d43cfd0, 0xbf284cd3, 0xac78bf27, 0x5e133c24, + 0x105ec76f, 0xe235446c, 0xf165b798, 0x030e349b, + 0xd7c45070, 0x25afd373, 0x36ff2087, 0xc494a384, + 0x9a879fa0, 0x68ec1ca3, 0x7bbcef57, 0x89d76c54, + 0x5d1d08bf, 0xaf768bbc, 0xbc267848, 0x4e4dfb4b, + 0x20bd8ede, 0xd2d60ddd, 0xc186fe29, 0x33ed7d2a, + 0xe72719c1, 0x154c9ac2, 0x061c6936, 0xf477ea35, + 0xaa64d611, 0x580f5512, 0x4b5fa6e6, 0xb93425e5, + 0x6dfe410e, 0x9f95c20d, 0x8cc531f9, 0x7eaeb2fa, + 0x30e349b1, 0xc288cab2, 0xd1d83946, 0x23b3ba45, + 0xf779deae, 0x05125dad, 0x1642ae59, 0xe4292d5a, + 0xba3a117e, 0x4851927d, 0x5b016189, 0xa96ae28a, + 0x7da08661, 0x8fcb0562, 0x9c9bf696, 0x6ef07595, + 0x417b1dbc, 0xb3109ebf, 0xa0406d4b, 0x522bee48, + 0x86e18aa3, 0x748a09a0, 0x67dafa54, 0x95b17957, + 0xcba24573, 0x39c9c670, 0x2a993584, 0xd8f2b687, + 0x0c38d26c, 0xfe53516f, 0xed03a29b, 0x1f682198, + 0x5125dad3, 0xa34e59d0, 0xb01eaa24, 0x42752927, + 0x96bf4dcc, 0x64d4cecf, 0x77843d3b, 0x85efbe38, + 0xdbfc821c, 0x2997011f, 0x3ac7f2eb, 0xc8ac71e8, + 0x1c661503, 0xee0d9600, 0xfd5d65f4, 0x0f36e6f7, + 0x61c69362, 0x93ad1061, 0x80fde395, 0x72966096, + 0xa65c047d, 0x5437877e, 0x4767748a, 0xb50cf789, + 0xeb1fcbad, 0x197448ae, 0x0a24bb5a, 0xf84f3859, + 0x2c855cb2, 0xdeeedfb1, 0xcdbe2c45, 0x3fd5af46, + 0x7198540d, 0x83f3d70e, 0x90a324fa, 0x62c8a7f9, + 0xb602c312, 0x44694011, 0x5739b3e5, 0xa55230e6, + 0xfb410cc2, 0x092a8fc1, 0x1a7a7c35, 0xe811ff36, + 0x3cdb9bdd, 0xceb018de, 0xdde0eb2a, 0x2f8b6829, + 0x82f63b78, 0x709db87b, 0x63cd4b8f, 0x91a6c88c, + 0x456cac67, 0xb7072f64, 0xa457dc90, 0x563c5f93, + 0x082f63b7, 0xfa44e0b4, 0xe9141340, 0x1b7f9043, + 0xcfb5f4a8, 0x3dde77ab, 0x2e8e845f, 0xdce5075c, + 0x92a8fc17, 0x60c37f14, 0x73938ce0, 0x81f80fe3, + 0x55326b08, 0xa759e80b, 0xb4091bff, 0x466298fc, + 0x1871a4d8, 0xea1a27db, 0xf94ad42f, 0x0b21572c, + 0xdfeb33c7, 0x2d80b0c4, 0x3ed04330, 0xccbbc033, + 0xa24bb5a6, 0x502036a5, 0x4370c551, 0xb11b4652, + 0x65d122b9, 0x97baa1ba, 0x84ea524e, 0x7681d14d, + 0x2892ed69, 0xdaf96e6a, 0xc9a99d9e, 0x3bc21e9d, + 0xef087a76, 0x1d63f975, 0x0e330a81, 0xfc588982, + 0xb21572c9, 0x407ef1ca, 0x532e023e, 0xa145813d, + 0x758fe5d6, 0x87e466d5, 0x94b49521, 0x66df1622, + 0x38cc2a06, 0xcaa7a905, 0xd9f75af1, 0x2b9cd9f2, + 0xff56bd19, 0x0d3d3e1a, 0x1e6dcdee, 0xec064eed, + 0xc38d26c4, 0x31e6a5c7, 0x22b65633, 0xd0ddd530, + 0x0417b1db, 0xf67c32d8, 0xe52cc12c, 0x1747422f, + 0x49547e0b, 0xbb3ffd08, 0xa86f0efc, 0x5a048dff, + 0x8ecee914, 0x7ca56a17, 0x6ff599e3, 0x9d9e1ae0, + 0xd3d3e1ab, 0x21b862a8, 0x32e8915c, 0xc083125f, + 0x144976b4, 0xe622f5b7, 0xf5720643, 0x07198540, + 0x590ab964, 0xab613a67, 0xb831c993, 0x4a5a4a90, + 0x9e902e7b, 0x6cfbad78, 0x7fab5e8c, 0x8dc0dd8f, + 0xe330a81a, 0x115b2b19, 0x020bd8ed, 0xf0605bee, + 0x24aa3f05, 0xd6c1bc06, 0xc5914ff2, 0x37faccf1, + 0x69e9f0d5, 0x9b8273d6, 0x88d28022, 0x7ab90321, + 0xae7367ca, 0x5c18e4c9, 0x4f48173d, 0xbd23943e, + 0xf36e6f75, 0x0105ec76, 0x12551f82, 0xe03e9c81, + 0x34f4f86a, 0xc69f7b69, 0xd5cf889d, 0x27a40b9e, + 0x79b737ba, 0x8bdcb4b9, 0x988c474d, 0x6ae7c44e, + 0xbe2da0a5, 0x4c4623a6, 0x5f16d052, 0xad7d5351 +}; + +uint32_t crc32c(uint32_t crc, const unsigned char* ptr, unsigned size) +{ + unsigned i; + + for(i=0;i> 8); + + return crc; +} + /****************************************************************************/ /* path */ @@ -910,7 +1184,7 @@ int lock_lock(const char* file) int lock_unlock(int f) { /* - * Intentionally don't remove file the lock file. + * Intentionally don't remove the lock file. * Removing it just introduces race course with other process * that could have already opened it. */ diff --git a/util.h b/util.h index dd1e38a1..dc1eefeb 100644 --- a/util.h +++ b/util.h @@ -41,6 +41,10 @@ struct stream { int state_index; /**< Index of the handle causing a state change. */ unsigned handle_size; /**< Number of handles. */ struct stream_handle* handle; /**< Set of handles. */ + off_t offset; /**< Offset into the file. */ + off_t offset_uncached; /**< Offset into the file excluding the cached data. */ + uint32_t crc; /**< CRC of the file. */ + uint32_t crc_uncached; /**< CRC of the file excluding the cached data. */ }; /** @@ -92,6 +96,16 @@ int sfill(STREAM* s); */ int sflush(STREAM* s); +/** + * Gets the file pointer. + */ +int64_t stell(STREAM* s); + +/** + * Gets the CRC of the processed data. + */ +uint32_t scrc(STREAM* s); + /** * Checks if the buffer has enough data loaded. */ @@ -124,6 +138,14 @@ static inline int serror(STREAM* s) return s->state == STREAM_STATE_ERROR; } +/** + * Checks the eof status. Like feof(). + */ +static inline int seof(STREAM* s) +{ + return s->state == STREAM_STATE_EOF; +} + /** * Gets the index of the handle that caused the error. */ @@ -168,6 +190,12 @@ static inline void sungetc(int c, STREAM* s) --s->pos; } +/** + * Reads a fixed amount of chars. + * Returns 0 on success, or -1 on error. + */ +int sread(STREAM* f, void* void_data, unsigned size); + /** * Gets a char from a stream, ignoring one '\r'. */ @@ -240,6 +268,33 @@ int sgetu64(STREAM* f, uint64_t* value); */ int sgethex(STREAM* f, void* data, int size); +/****************************************************************************/ +/* binary get */ + +/** + * Reads a binary 32 bit number in packet format. + * Returns <0 if there isn't enough to read. + */ +int sgetb32(STREAM* f, uint32_t* value); + +/** + * Reads a binary 64 bit number in packet format. + * Returns <0 if there isn't enough to read. + */ +int sgetb64(STREAM* f, uint64_t* value); + +/** + * Reads a binary 32 bit number in little endian format. + * Returns <0 if there isn't enough to read. + */ +int sgetble32(STREAM* f, uint32_t* value); + +/** + * Reads a binary string. + * Returns -1 on error or if the buffer is too small, or the number of chars read. + */ +int sgetbs(STREAM* f, char* str, int size); + /****************************************************************************/ /* put */ @@ -306,6 +361,33 @@ int sputu64(uint64_t value, STREAM* s); */ int sputhex(const void* void_data, int size, STREAM* s); +/****************************************************************************/ +/* binary put */ + +/** + * Writes a binary 32 bit number in packed format. + * Returns 0 on success or -1 on error. + */ +int sputb32(uint32_t value, STREAM* s); + +/** + * Writes a binary 64 bit number in packed format. + * Returns 0 on success or -1 on error. + */ +int sputb64(uint64_t value, STREAM* s); + +/** + * Writes a binary 32 bit number in little endian format. + * Returns 0 on success or -1 on error. + */ +int sputble32(uint32_t value, STREAM* s); + +/** + * Writes a binary string. + * Returns 0 on success or -1 on error. + */ +int sputbs(const char* str, STREAM* s); + /****************************************************************************/ /* path */ @@ -411,6 +493,14 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* */ void randomize(void* ptr, unsigned size); +/****************************************************************************/ +/* crc */ + +/** + * Computes the CRC-32 (Castagnoli) + */ +uint32_t crc32c(uint32_t crc, const unsigned char* ptr, unsigned size); + /****************************************************************************/ /* sax */