From 6178b1be83a727eb3a5ed2d2d5dc787fc71423d7 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 22 Sep 2013 23:54:19 +0200 Subject: [PATCH 1/6] Fixes the computation of the blockmax when importing a text file. --- state.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/state.c b/state.c index 1646871c..214ec75c 100644 --- a/state.c +++ b/state.c @@ -1113,10 +1113,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE deleted->block.parity_pos = v_pos; - /* keep track of the max block number used in parity */ - if (v_pos + 1 > blockmax) - blockmax = v_pos + 1; - /* read the hash */ c = sgetc(f); if (c != ' ') { From efd8a3c01a1fe070695436d1416f5db7894f7a7b Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 22 Sep 2013 23:54:49 +0200 Subject: [PATCH 2/6] Fixes a typo in messages. --- elem.c | 2 +- rehash.c | 2 +- state.c | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/elem.c b/elem.c index 75834998..e3735763 100644 --- a/elem.c +++ b/elem.c @@ -583,7 +583,7 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) /* if there is a deleted block, the disk is not empty */ return 0; default: - fprintf(stderr, "Internal incosistency for used block in disk '%s' without files\n", disk->name); + fprintf(stderr, "Internal inconsistency for used block in disk '%s' without files\n", disk->name); exit(EXIT_FAILURE); } } diff --git a/rehash.c b/rehash.c index c557b507..261d7e2b 100644 --- a/rehash.c +++ b/rehash.c @@ -66,7 +66,7 @@ void state_rehash(struct snapraid_state* state) } if (info_get_rehash(info)) { - fprintf(stderr, "Internal incosistency for a rehash already in progress\n"); + fprintf(stderr, "Internal inconsistency for a rehash already in progress\n"); exit(EXIT_FAILURE); } diff --git a/state.c b/state.c index 214ec75c..8e5d39a6 100644 --- a/state.c +++ b/state.c @@ -768,7 +768,7 @@ static void state_map(struct snapraid_state* state) } if (j==0) { - fprintf(stderr, "Internal incosistency for mapping '%s'\n", map->name); + fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name); exit(EXIT_FAILURE); } @@ -1073,7 +1073,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE rehash = 1; if (state->prevhash == HASH_UNDEFINED) { - fprintf(stderr, "Internal incosistency for missing previous checksum in '%s' at line %u\n", path, line); + fprintf(stderr, "Internal inconsistency for missing previous checksum in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } } else { @@ -1632,7 +1632,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE /* check that the stored parity size matches the loaded state */ if (blockmax != parity_size(state)) { - fprintf(stderr, "Internal incosistency in parity size in '%s' at line %u\n", path, line); + fprintf(stderr, "Internal inconsistency in parity size in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } state->loaded_blockmax = blockmax; @@ -1740,7 +1740,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) /* find the disk for this mapping */ disk = find_disk(state, map->name); if (!disk) { - fprintf(stderr, "Internal incosistency for unmapped disk '%s'\n", map->name); + fprintf(stderr, "Internal inconsistency for unmapped disk '%s'\n", map->name); exit(EXIT_FAILURE); } @@ -2299,7 +2299,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST if (rehash && state->prevhash == HASH_UNDEFINED) { decoding_error(path, f); - fprintf(stderr, "Internal incosistency for missing previous checksum!\n"); + fprintf(stderr, "Internal inconsistency for missing previous checksum!\n"); exit(EXIT_FAILURE); } @@ -2316,13 +2316,13 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST if (position_has_any_hash(state, v_pos)) { if (!info) { decoding_error(path, f); - fprintf(stderr, "Internal incosistency for missing info!\n"); + fprintf(stderr, "Internal inconsistency for missing info!\n"); exit(EXIT_FAILURE); } } else { if (info) { decoding_error(path, f); - fprintf(stderr, "Internal incosistency for unexpected info!\n"); + fprintf(stderr, "Internal inconsistency for unexpected info!\n"); exit(EXIT_FAILURE); } } @@ -2683,7 +2683,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* check that the stored parity size matches the loaded state */ if (blockmax != parity_size(state)) { - fprintf(stderr, "Internal incosistency in parity size in '%s' at offset %"PRIi64"\n", path, stell(f)); + fprintf(stderr, "Internal inconsistency in parity size in '%s' at offset %"PRIi64"\n", path, stell(f)); exit(EXIT_FAILURE); } state->loaded_blockmax = blockmax; @@ -2804,7 +2804,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* find the disk for this mapping */ disk = find_disk(state, map->name); if (!disk) { - fprintf(stderr, "Internal incosistency for unmapped disk '%s'\n", map->name); + fprintf(stderr, "Internal inconsistency for unmapped disk '%s'\n", map->name); exit(EXIT_FAILURE); } From 593d4fd5f1013dd21189e7cae165551c9417dbbf Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 23 Sep 2013 22:11:21 +0200 Subject: [PATCH 3/6] Updates HISTORY. --- HISTORY | 1 + 1 file changed, 1 insertion(+) diff --git a/HISTORY b/HISTORY index 4b8f63a7..a62bc688 100644 --- a/HISTORY +++ b/HISTORY @@ -11,6 +11,7 @@ SnapRAID HISTORY Just in case that you lose everything, except the content file. * At the end of sync/scrub/check/fix prints "Everything OK" if no error was found. This should make clear that everything is really OK. + * Fixes importing of old text content files from an incomplete sync. 4.0 2013/9 ========== From 90b4efdbba79e523898d56f36ac7063650536f8e Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 23 Sep 2013 22:44:57 +0200 Subject: [PATCH 4/6] Updates the HISTORY. --- HISTORY | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/HISTORY b/HISTORY index a62bc688..5da8642b 100644 --- a/HISTORY +++ b/HISTORY @@ -4,14 +4,15 @@ SnapRAID HISTORY 4.1 2013/9 ========== * If the underline filesystem doesn't support the FIEMAP command, - fallback to use the inode number for sorting files. - This happens with reiserfs in Linux. + automatically fallback to use the inode number for sorting files. + This happens with reiserfs and FUSE in Linux. + * Fixes the import of content files from previous version of SnapRAID that + are the result of an incomplete sync. * Added a new -C, --gen-conf option to generate a dummy configuration file from the info in the content file. Just in case that you lose everything, except the content file. * At the end of sync/scrub/check/fix prints "Everything OK" if no error was found. This should make clear that everything is really OK. - * Fixes importing of old text content files from an incomplete sync. 4.0 2013/9 ========== @@ -29,7 +30,7 @@ SnapRAID HISTORY This happens only if the file has the same path, size and timestamp at nanosecond precision. * Added a hash seed to make harder intentional collision attacks. - * When insering files for the first time, sort them by their physical + * When inserting files for the first time, sort them by their physical address to improve read performance. * Optimized the cache use for the all the RAID computations. This improves a lot the RAID performance. From ef969263cdfc4896857520273482cd2356ae6c9b Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 23 Sep 2013 23:58:19 +0200 Subject: [PATCH 5/6] Fallback to FIBMAP if FIEMAP doesn't work. --- HISTORY | 4 ++-- unix.c | 45 +++++++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 14 deletions(-) diff --git a/HISTORY b/HISTORY index 5da8642b..8f670868 100644 --- a/HISTORY +++ b/HISTORY @@ -4,8 +4,8 @@ SnapRAID HISTORY 4.1 2013/9 ========== * If the underline filesystem doesn't support the FIEMAP command, - automatically fallback to use the inode number for sorting files. - This happens with reiserfs and FUSE in Linux. + automatically fallback to use FIBMAP and then the inode number for sorting + files. * Fixes the import of content files from previous version of SnapRAID that are the result of an incomplete sync. * Added a new -C, --gen-conf option to generate a dummy configuration diff --git a/unix.c b/unix.c index ba1dfbee..abda84b3 100644 --- a/unix.c +++ b/unix.c @@ -129,43 +129,64 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) /* In Linux gets the real physical address of the file */ /* Note that FIEMAP doesn't require root permission */ int f; - struct { struct fiemap fiemap; struct fiemap_extent extent; } fm; + unsigned int blknum; f = open(path, O_RDONLY); if (f == -1) { return -1; } + /* first try with FIEMAP */ memset(&fm, 0, sizeof(fm)); fm.fiemap.fm_start = 0; fm.fiemap.fm_length = ~0ULL; fm.fiemap.fm_flags = FIEMAP_FLAG_SYNC; /* required to ensure that just created files report a valid address and not 0 */ fm.fiemap.fm_extent_count = 1; /* we are interested only at the first block */ - if (ioctl(f, FS_IOC_FIEMAP, &fm) == -1) { - /* if the underline filesystem doesn't support FS_IOC_FIEMAP */ - /* fallback to use the inode */ - if (errno == ENOTSUP) { - close(f); - *physical = st->st_ino; - return 0; - } + if (ioctl(f, FS_IOC_FIEMAP, &fm) == 0) { + *physical = fm.fiemap.fm_extents[0].fe_physical; + if (close(f) == -1) + return -1; + return 0; + } + /* if it's an unexpected error fails */ + if (errno != ENOTSUP) { close(f); return -1; } - *physical = fm.fiemap.fm_extents[0].fe_physical; + /* if empty, FIBMAP doesn't work */ + if (st->st_size == 0) { + *physical = 0; + if (close(f) == -1) + return -1; + return 0; + } - if (close(f) == -1) { + /* then try with FIBMAP */ + blknum = 0; /* first block */ + if (ioctl(f, FIBMAP, &blknum) == 0) { + *physical = blknum; + if (close(f) == -1) + return -1; + return 0; + } + + /* if it's an unexpected error fails */ + if (errno != ENOTSUP && errno != EPERM) { + close(f); return -1; } - (void)st; /* not used here */ + /* otherwise uses the inode */ + *physical = st->st_ino; + if (close(f) == -1) + return -1; #else /* In a generic Unix uses the inode as fake physical address */ *physical = st->st_ino; From 4eee6b9c4ab35c76a376eccca5296dc729c8d256 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 24 Sep 2013 21:45:07 +0200 Subject: [PATCH 6/6] Doesn't fail in filephy() after an unexpected error. There are too many filesystem to know that are expected errors. --- scan.c | 2 ++ unix.c | 21 +++++++-------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/scan.c b/scan.c index 34c51eb2..8aa58abc 100644 --- a/scan.c +++ b/scan.c @@ -919,6 +919,8 @@ void state_scan(struct snapraid_state* state, int output) } /* sort the files before inserting them */ + /* we use a stable sort to ensure that if the reported physical offset/inode */ + /* are always 0, we keep at least the directory order */ switch (state->opt.force_order) { case SORT_PHYSICAL : tommy_list_sort(&scan->file_insert_list, file_physical_compare); diff --git a/unix.c b/unix.c index abda84b3..f60b9d07 100644 --- a/unix.c +++ b/unix.c @@ -141,6 +141,7 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) } /* first try with FIEMAP */ + /* if works for ext2, ext3, ext4, xfs */ memset(&fm, 0, sizeof(fm)); fm.fiemap.fm_start = 0; fm.fiemap.fm_length = ~0ULL; @@ -154,12 +155,6 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) return 0; } - /* if it's an unexpected error fails */ - if (errno != ENOTSUP) { - close(f); - return -1; - } - /* if empty, FIBMAP doesn't work */ if (st->st_size == 0) { *physical = 0; @@ -169,6 +164,8 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) } /* then try with FIBMAP */ + /* it works for jfs, reiserfs, ntfs-3g */ + /* in exfat it always returns 0, that it's anyway better than the fake inodes */ blknum = 0; /* first block */ if (ioctl(f, FIBMAP, &blknum) == 0) { *physical = blknum; @@ -177,14 +174,10 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) return 0; } - /* if it's an unexpected error fails */ - if (errno != ENOTSUP && errno != EPERM) { - close(f); - return -1; - } - - /* otherwise uses the inode */ - *physical = st->st_ino; + /* otherwise don't use anything, and keep the existing order */ + /* at now this should happen only for vfat */ + /* and it's surely better than using fake inodes */ + *physical = 0; if (close(f) == -1) return -1; #else