diff --git a/HISTORY b/HISTORY index 4a171eba..9c984ac9 100644 --- a/HISTORY +++ b/HISTORY @@ -14,8 +14,7 @@ SnapRAID HISTORY 4.1 2013/9 (BETA) ================= * If the underline filesystem doesn't support the FIEMAP command, - automatically fallback to use FIBMAP and then the inode number for sorting - files. + automatically fallback to use FIBMAP for sorting files. * Fixes the import of content files from previous version of SnapRAID that are the result of an incomplete sync. * Added a new -C, --gen-conf option to generate a dummy configuration diff --git a/elem.c b/elem.c index e3735763..b3e7735e 100644 --- a/elem.c +++ b/elem.c @@ -524,6 +524,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev disk->device = dev; disk->first_free_block = 0; disk->has_not_persistent_inodes = 0; + disk->has_not_reliable_physical = 0; disk->mapping = -1; tommy_list_init(&disk->filelist); tommy_list_init(&disk->deletedlist); diff --git a/elem.h b/elem.h index 1b045763..7c832a5a 100644 --- a/elem.h +++ b/elem.h @@ -274,6 +274,7 @@ struct snapraid_disk { uint64_t device; /**< Device identifier. */ block_off_t first_free_block; /**< First free searching block. */ int has_not_persistent_inodes; /**< If the underline filesystem has dynamic inodes. */ + int has_not_reliable_physical; /**< If the physical offset of files has duplicates. */ int mapping; /**< Index in the mapping vector. Used only as buffer when writing the content file. */ /**< diff --git a/scan.c b/scan.c index 8aa58abc..8b17ddda 100644 --- a/scan.c +++ b/scan.c @@ -277,14 +277,16 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* sub, const struct stat* st, uint64_t physical) { struct snapraid_file* file; + uint64_t inode; file = 0; /* - * If the disk has persistent inodes, try a search by inode. + * If the disk has persistent inodes, try a search by inode, + * to detect moved files. * * For persistent inodes we mean inodes that keep their values when the filesystem - * is unmounted and remounted. This don't always happen. + * is unmounted and remounted. This don't always happen. * * Cases found are: * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/. @@ -293,15 +295,18 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, * making inode collision more easy (exFAT by design supports 10ms precision). * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount. * - * In such cases, to avoid possible random collisions, it's better to just avoid - * to use inodes at all. + * In such cases, to avoid possible random collisions, it's better to disable the moved + * file recognizition. + * + * We do this implicitely removing all the inode before searching for files. + * This ensure that no file is found with an old inode, but at the same time, + * it allows to find new files with the same inode, and to identify them as hardlinks. */ - if (!disk->has_not_persistent_inodes) { - /* first try finding by inode */ - uint64_t inode = st->st_ino; - file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); - } + inode = st->st_ino; + file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode)); + + /* identify moved files searching by inode */ if (file) { /* check if the file is not changed */ if (file->size == st->st_size @@ -816,6 +821,9 @@ void state_scan(struct snapraid_state* state, int output) tommy_node* node; int ret; int has_persistent_inode; + unsigned phy_count; + unsigned phy_dup; + uint64_t phy_last; scan = malloc_nofail(sizeof(struct snapraid_scan)); scan->count_equal = 0; @@ -838,10 +846,29 @@ void state_scan(struct snapraid_state* state, int output) fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno)); exit(EXIT_FAILURE); } - /* mark the disk if required */ if (!has_persistent_inode) { - fprintf(stderr, "WARNING! Inodes use disabled for disk '%s' because they are not persistent.\n", disk->dir); + /* mask the disk */ disk->has_not_persistent_inodes = 1; + + /* removes all the inodes from the inode collection */ + /* if they are not persistent, all of them could be changed now */ + /* and we don't want to find false matching ones */ + /* see scan_file() for more details */ + node = disk->filelist; + while (node) { + struct snapraid_file* file = node->data; + + node = node->next; + + /* remove from the inode set */ + tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); + + /* clear the inode */ + file->inode = 0; + + /* mark as missing inode */ + file_flag_set(file, FILE_IS_WITHOUT_INODE); + } } scan_dir(scan, state, output, disk, disk->dir, ""); @@ -939,9 +966,20 @@ void state_scan(struct snapraid_state* state, int output) /* insert all the new files, we insert them only after the deletion */ /* to reuse the just freed space */ node = scan->file_insert_list; + phy_count = 0; + phy_dup = 0; + phy_last = -1; while (node) { struct snapraid_file* file = node->data; + /* if the file is not empty, count duplicate physical offsets */ + if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) { + if (phy_count > 0 && file->physical == phy_last) + ++phy_dup; + phy_last = file->physical; + ++phy_count; + } + /* next node */ node = node->next; @@ -949,6 +987,12 @@ void state_scan(struct snapraid_state* state, int output) scan_file_insert(state, disk, file); } + /* mark the disk without reliable physical offset if it has duplicates */ + /* here it should never happen because we already sorted out hardlinks */ + if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) { + disk->has_not_reliable_physical = 1; + } + /* insert all the new links */ node = scan->link_insert_list; while (node) { @@ -976,21 +1020,21 @@ void state_scan(struct snapraid_state* state, int output) /* checks for disks where all the previously existing files where removed */ if (!state->opt.force_empty) { - int has_empty = 0; + int done = 0; for(i=state->disklist,j=scanlist;i!=0;i=i->next,j=j->next) { struct snapraid_disk* disk = i->data; struct snapraid_scan* scan = j->data; if (scan->count_equal == 0 && scan->count_move == 0 && scan->count_restore == 0 && (scan->count_remove != 0 || scan->count_change != 0)) { - if (!has_empty) { - has_empty = 1; + if (!done) { + done = 1; fprintf(stderr, "All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir); } else { fprintf(stderr, ", disk '%s' at dir '%s'", disk->name, disk->dir); } } } - if (has_empty) { + if (done) { fprintf(stderr, " are now missing or rewritten!\n"); fprintf(stderr, "This could happen when deleting all the files from a disk,\n"); fprintf(stderr, "and restoring them with a program not setting correctly the timestamps.\n"); @@ -1001,6 +1045,46 @@ void state_scan(struct snapraid_state* state, int output) } } + /* checks for disks without the physical offset support */ + if (state->opt.force_order == SORT_PHYSICAL) { + int done = 0; + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* disk = i->data; + + if (disk->has_not_reliable_physical) { + if (!done) { + done = 1; + fprintf(stderr, "WARNING! Physical offsets not supported for disk '%s'", disk->name); + } else { + fprintf(stderr, ", '%s", disk->name); + } + } + } + if (done) { + fprintf(stderr, ". Performance won't be optimal.\n"); + } + } + + /* Check for disks without persisten inodes */ + { + int done = 0; + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* disk = i->data; + + if (disk->has_not_persistent_inodes) { + if (!done) { + done = 1; + fprintf(stderr, "WARNING! Inodes are not persistent for disk '%s'", disk->name); + } else { + fprintf(stderr, ", '%s", disk->name); + } + } + } + if (done) { + fprintf(stderr, ". Move operations won't be optimized.\n"); + } + } + if (state->opt.verbose || output) { struct snapraid_scan total; diff --git a/scrub.c b/scrub.c index d525c671..9965feda 100644 --- a/scrub.c +++ b/scrub.c @@ -101,10 +101,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par continue; } - /* if we reached the count limit */ - if (countmax >= countlimit) { - /* skip it */ - continue; + /* if the time is less than the limit, always include */ + /* otherwise, check if we reaced the count max */ + if (blocktime == timelimit) { + /* if we reached the count limit */ + if (countmax >= countlimit) { + /* skip it */ + continue; + } } } @@ -154,10 +158,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par continue; } - /* if we reached the count limit */ - if (recountmax >= countlimit) { - /* skip it */ - continue; + /* if the time is less than the limit, always include */ + /* otherwise, check if we reaced the count max */ + if (blocktime == timelimit) { + /* if we reached the count limit */ + if (recountmax >= countlimit) { + /* skip it */ + continue; + } } } @@ -439,7 +447,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) recentlimit = now; } else if (percentage != -1 || olderthan != -1) { if (percentage != -1) - countlimit = blockmax * percentage / 100; + countlimit = (blockmax * percentage + 99) / 100; else countlimit = blockmax; if (olderthan != -1) @@ -488,8 +496,9 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) timelimit = info_get_time(infomap[countlimit - 1]); /* don't scrub too recent blocks */ - if (timelimit > recentlimit) + if (timelimit > recentlimit) { timelimit = recentlimit; + } } else { /* we don't have a time limit based on quota */ timelimit = recentlimit; diff --git a/snapraid.1 b/snapraid.1 index ae6d0ddb..1cf6c8cf 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -791,14 +791,14 @@ memory to run SnapRAID. As a rule of thumb, with 4 GiB or more memory use the default 256, with 2 GiB use 512, and with 1 GiB use 1024. .PP -In more details SnapRAID requires about TS*24/BS bytes +In more details SnapRAID requires about TS*28/BS bytes of RAM memory to run. Where TS is the total size in bytes of your disk array, and BS is the block size in bytes. .PP -For example with 6 disk of 2 TiB and a block size of 256 KiB +For example with 4 disk of 3 TiB and a block size of 256 KiB (1 KiB = 1024 Bytes) you have: .PP -RAM = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB +RAM = (4 * 3 * 2^40) * 28 / (256 * 2^10) = 1.4 GiB .PD 0 .PP .PD diff --git a/snapraid.d b/snapraid.d index 3b55c957..1e37c502 100644 --- a/snapraid.d +++ b/snapraid.d @@ -623,14 +623,14 @@ Configuration As a rule of thumb, with 4 GiB or more memory use the default 256, with 2 GiB use 512, and with 1 GiB use 1024. - In more details SnapRAID requires about TS*24/BS bytes + In more details SnapRAID requires about TS*28/BS bytes of RAM memory to run. Where TS is the total size in bytes of your disk array, and BS is the block size in bytes. - For example with 6 disk of 2 TiB and a block size of 256 KiB + For example with 4 disk of 3 TiB and a block size of 256 KiB (1 KiB = 1024 Bytes) you have: - :RAM = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB + :RAM = (4 * 3 * 2^40) * 28 / (256 * 2^10) = 1.4 GiB You could instead decrease this value if you have a lot of small files in the disk array. For each file, even if of few diff --git a/snapraid.txt b/snapraid.txt index f7397580..5d5e6263 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -683,14 +683,14 @@ memory to run SnapRAID. As a rule of thumb, with 4 GiB or more memory use the default 256, with 2 GiB use 512, and with 1 GiB use 1024. -In more details SnapRAID requires about TS*24/BS bytes +In more details SnapRAID requires about TS*28/BS bytes of RAM memory to run. Where TS is the total size in bytes of your disk array, and BS is the block size in bytes. -For example with 6 disk of 2 TiB and a block size of 256 KiB +For example with 4 disk of 3 TiB and a block size of 256 KiB (1 KiB = 1024 Bytes) you have: -RAM = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB +RAM = (4 * 3 * 2^40) * 28 / (256 * 2^10) = 1.4 GiB You could instead decrease this value if you have a lot of small files in the disk array. For each file, even if of few diff --git a/state.c b/state.c index ac5cc3e2..4de1d3a0 100644 --- a/state.c +++ b/state.c @@ -640,7 +640,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co /* no previous hash by default */ state->prevhash = HASH_UNDEFINED; - /* intentionally not set the prevhashseed */ + /* intentionally not set the prevhashseed, if used valgrind will warn about it */ if (state->opt.gui) { tommy_node* i; @@ -1718,7 +1718,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) if (info_has_rehash) { if (state->prevhash == HASH_MURMUR3) { sputsl("prevchecksum murmur3", f); - } else if (state->hash == HASH_SPOOKY2) { + } else if (state->prevhash == HASH_SPOOKY2) { sputsl("prevchecksum spooky2", f); } else { fprintf(stderr, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); @@ -2783,7 +2783,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) sputc('C', f); if (state->prevhash == HASH_MURMUR3) { sputc('u', f); - } else if (state->hash == HASH_SPOOKY2) { + } else if (state->prevhash == HASH_SPOOKY2) { sputc('k', f); } else { fprintf(stderr, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); @@ -3185,7 +3185,8 @@ void state_read(struct snapraid_state* state) /* previous hash, start with an undefined value */ state->prevhash = HASH_UNDEFINED; - memset(state->prevhashseed, 0, HASH_SIZE); + + /* intentionally not set the prevhashseed, if used valgrind will warn about it */ /* get the first char to detect the file type */ c = sgetc(f); diff --git a/status.c b/status.c index a9ba7565..ade7f28f 100644 --- a/status.c +++ b/status.c @@ -249,7 +249,11 @@ int state_status(struct snapraid_state* state) if (rehash) { printf("You have a rehash in progress at %u%%.\n", (count - rehash) * 100 / count); } else { - printf("No rehash is in progress.\n"); + if (state->besthash != state->hash) { + printf("No rehash is in progress, but for optimal performance one is recommended.\n"); + } else { + printf("No rehash is in progress or needed.\n"); + } } if (bad) { diff --git a/unix.c b/unix.c index f60b9d07..a775b9da 100644 --- a/unix.c +++ b/unix.c @@ -141,14 +141,14 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) } /* first try with FIEMAP */ - /* if works for ext2, ext3, ext4, xfs */ + /* if works for ext2, ext3, ext4, xfs, btrfs */ memset(&fm, 0, sizeof(fm)); fm.fiemap.fm_start = 0; fm.fiemap.fm_length = ~0ULL; fm.fiemap.fm_flags = FIEMAP_FLAG_SYNC; /* required to ensure that just created files report a valid address and not 0 */ fm.fiemap.fm_extent_count = 1; /* we are interested only at the first block */ - if (ioctl(f, FS_IOC_FIEMAP, &fm) == 0) { + if (ioctl(f, FS_IOC_FIEMAP, &fm) != -1) { *physical = fm.fiemap.fm_extents[0].fe_physical; if (close(f) == -1) return -1; @@ -167,7 +167,7 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) /* it works for jfs, reiserfs, ntfs-3g */ /* in exfat it always returns 0, that it's anyway better than the fake inodes */ blknum = 0; /* first block */ - if (ioctl(f, FIBMAP, &blknum) == 0) { + if (ioctl(f, FIBMAP, &blknum) != -1) { *physical = blknum; if (close(f) == -1) return -1; @@ -181,10 +181,12 @@ int filephy(const char* path, struct stat* st, uint64_t* physical) if (close(f) == -1) return -1; #else - /* In a generic Unix uses the inode as fake physical address */ - *physical = st->st_ino; + /* In a generic Unix use a dummy value for all the files */ + /* We don't want to risk to use the inode without knowing if it really improves performance */ + *physical = 0; (void)path; /* not used here */ + (void)st; #endif return 0;