From f778fb691f96827610ec8f1d93bae7020d946a74 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 29 Oct 2017 18:07:01 +0100 Subject: [PATCH] Apply new hardlink behaviour also to NTFS in nix platforms This is a specific NTFS behaviour, not limited to Windows. --- HISTORY | 8 +++--- cmdline/elem.c | 1 + cmdline/elem.h | 1 + cmdline/mingw.c | 13 ++++++++- cmdline/mingw.h | 3 ++- cmdline/portable.h | 2 +- cmdline/scan.c | 67 +++++++++++++++++++++++++++++++--------------- cmdline/state.c | 4 +-- cmdline/unix.c | 22 +++++++++++++-- 9 files changed, 88 insertions(+), 33 deletions(-) diff --git a/HISTORY b/HISTORY index bdb7dbce..317a98a8 100644 --- a/HISTORY +++ b/HISTORY @@ -1,12 +1,12 @@ SnapRAID HISTORY ================ -11.2 2017/10 +11.2 2017/11 ============ - * Fix recognition of Windows hardlinks. They behave differently than - Unix hardlinks and this could result in SnapRAID reporting + * Fix recognition of NTFS hardlinks. They behave differently than + standard Unix hardlinks and this could result in SnapRAID reporting internal incosistency errors for detecting links to the same file - with different attributes. + with different metadata attributes. * More efficient 'pool' command that updates only the links that need to be updated. This ensures that no change is done, avoiding to trigger a directory rescan of other programs. diff --git a/cmdline/elem.c b/cmdline/elem.c index 4add9246..eb663ace 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -760,6 +760,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev disk->free_blocks = 0; disk->first_free_block = 0; disk->has_volatile_inodes = 0; + disk->has_volatile_hardlinks = 0; disk->has_unreliable_physical = 0; disk->has_different_uuid = 0; disk->has_unsupported_uuid = *uuid == 0; /* empty UUID means unsupported */ diff --git a/cmdline/elem.h b/cmdline/elem.h index 00a8b8f3..c41cdaeb 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -348,6 +348,7 @@ struct snapraid_disk { block_off_t first_free_block; int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */ + int has_volatile_hardlinks; /**< If the underline file-system has not syncronized metadata for hardlink (NTFS). */ int has_unreliable_physical; /**< If the physical offset of files has duplicates. */ int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */ int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */ diff --git a/cmdline/mingw.c b/cmdline/mingw.c index f14579eb..0b157b6a 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -501,6 +501,8 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ st->st_ino <<= 32; st->st_ino |= info->nFileIndexLow; + st->st_nlink = info->nNumberOfLinks; + st->st_dev = info->dwVolumeSerialNumber; /* GetFileInformationByHandle() ensures to return synced information */ @@ -553,6 +555,8 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL st->st_ino = stream->FileId.QuadPart; + st->st_nlink = info->nNumberOfLinks; + st->st_dev = info->dwVolumeSerialNumber; /* directory listing doesn't ensure to return synced information */ @@ -598,6 +602,9 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s /* No inode information available */ st->st_ino = 0; + /* No link information available */ + st->st_nlink = 0; + /* No device information available */ st->st_dev = 0; @@ -1705,7 +1712,7 @@ int filephy(const char* file, uint64_t size, uint64_t* physical) return 0; } -int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space) +int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_hardlinks, uint64_t* total_space, uint64_t* free_space) { wchar_t conv_buf[CONV_MAX]; @@ -1713,6 +1720,10 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, u if (has_persistent_inode) *has_persistent_inode = 1; + /* NTFS doesn't syncronize hardlinks metadata */ + if (has_syncronized_hardlinks) + *has_syncronized_hardlinks = 0; + if (free_space || total_space) { ULARGE_INTEGER total_bytes; ULARGE_INTEGER total_free_bytes; diff --git a/cmdline/mingw.h b/cmdline/mingw.h index 9cebf6e9..7ac80e19 100644 --- a/cmdline/mingw.h +++ b/cmdline/mingw.h @@ -94,7 +94,7 @@ #define dirent_hidden windows_dirent_hidden #define HAVE_STRUCT_DIRENT_D_STAT 1 #undef HAVE_STRUCT_DIRENT_D_INO -#undef HAVE_STRUCT_STAT_ST_NLINK +#define HAVE_STRUCT_STAT_ST_NLINK 1 #define dirent_lstat windows_dirent_lstat #define stat_desc windows_stat_desc #undef sleep @@ -148,6 +148,7 @@ struct windows_stat { int32_t st_mtimensec; uint32_t st_mode; uint32_t st_dev; + uint32_t st_nlink; int st_hidden; const char* st_desc; int st_sync; /**< If the information are in sync with the file-system. */ diff --git a/cmdline/portable.h b/cmdline/portable.h index 059cd38c..053ad857 100644 --- a/cmdline/portable.h +++ b/cmdline/portable.h @@ -345,7 +345,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical); * Check if the underline file-system support persistent inodes. * Return -1 on error, 0 on success. */ -int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space); +int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_hardlinks, uint64_t* total_space, uint64_t* free_space); /** * Get the tick counter value. diff --git a/cmdline/scan.c b/cmdline/scan.c index af59e7fd..d7b86c75 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -501,11 +501,16 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc * because the metadata in the directory is updated only when the file * is closed. * + * The same happens for hardlinks that duplicate metatada. + * The link metadata is updated only when the link is opened. + * This extends also to st_size and st_nlink. + * * See also: * Why is the file size reported incorrectly for files that are still being written to? * http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx */ - log_fatal("WARNING! Detected uncached time change for file '%s'\n", sub); + log_fatal("WARNING! Detected uncached time change from %" PRIu64 ".%09u to %" PRIu64 ".%09u for file '%s'\n", + (uint64_t)st->st_mtime, (uint32_t)st->st_mtimensec, (uint64_t)synced_st.st_mtime, (uint32_t)synced_st.st_mtimensec, sub); log_fatal("It's better if you run SnapRAID without other processes running.\n"); #endif st->st_mtime = synced_st.st_mtime; @@ -514,15 +519,26 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc if (st->st_size != synced_st.st_size) { #ifndef _WIN32 - log_fatal("WARNING! Detected uncached size change for file '%s'\n", sub); + log_fatal("WARNING! Detected uncached size change from %" PRIu64 " to %" PRIu64 " for file '%s'\n", + (uint64_t)st->st_size, (uint64_t)synced_st.st_size, sub); log_fatal("It's better if you run SnapRAID without other processes running.\n"); #endif st->st_size = synced_st.st_size; } + if (st->st_nlink != synced_st.st_nlink) { +#ifndef _WIN32 + log_fatal("WARNING! Detected uncached nlink change from %u to %u for file '%s'\n", + (uint32_t)st->st_nlink , (uint32_t)synced_st.st_nlink, sub); + log_fatal("It's better if you run SnapRAID without other processes running.\n"); +#endif + st->st_nlink = synced_st.st_nlink; + } + if (st->st_ino != synced_st.st_ino) { - log_fatal("DANGER! Detected uncached inode change for file '%s'\n", sub); - log_fatal("Please ensure to run SnapRAID without other processes running.\n"); + log_fatal("DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n", + (uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub); + log_fatal("It's better if you run SnapRAID without other processes running.\n"); /* at this point, it's too late to change inode */ /* and having inconsistent inodes may result to internal failures */ /* so, it's better to abort */ @@ -666,14 +682,14 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, ) { /* check if multiple files have the same inode */ if (file_flag_has(file, FILE_IS_PRESENT)) { -#if HAVE_STRUCT_STAT_ST_NLINK - if (st->st_nlink <= 1) { + /* if has_volatile_hardlinks is true, the nlink value is not reliable */ + if (!disk->has_volatile_hardlinks && st->st_nlink == 1) { /* LCOV_EXCL_START */ log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub); os_abort(); /* LCOV_EXCL_STOP */ } -#endif + /* it's a hardlink */ scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK); return; @@ -749,20 +765,23 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, * ...both time and size of A and B don't match! */ if (file_flag_has(file, FILE_IS_PRESENT)) { -#ifdef _WIN32 - /* "suppose" a hardlink */ + /* if has_volatile_hardlinks is true, the nlink value is not reliable */ + if (!disk->has_volatile_hardlinks && st->st_nlink == 1) { + /* LCOV_EXCL_START */ + log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different attributes: size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d\n", + file->inode, disk->dir, sub, disk->dir, file->sub, + file->size, (uint64_t)st->st_size, + file->mtime_sec, (uint64_t)st->st_mtime, + file->mtime_nsec, STAT_NSEC(st)); + os_abort(); + /* LCOV_EXCL_STOP */ + } + + /* LCOV_EXCL_START */ + /* suppose it's hardlink with not synced metadata */ scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK); return; -#else - /* LCOV_EXCL_START */ - log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different (size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d)\n", - file->inode, disk->dir, sub, disk->dir, file->sub, - file->size, (uint64_t)st->st_size, - file->mtime_sec, (uint64_t)st->st_mtime, - file->mtime_nsec, STAT_NSEC(st)); - os_abort(); /* LCOV_EXCL_STOP */ -#endif } /* assume a previously used inode, it's the worst case */ @@ -1351,7 +1370,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch #if HAVE_LSTAT_SYNC /* if the st_ino field is missing, takes care to fill it using the extended lstat() */ /* this can happen only in Windows */ - if (st->st_ino == 0) { + if (st->st_ino == 0 || st->st_nlink == 0) { if (lstat_sync(path_next, st, 0) != 0) { /* LCOV_EXCL_START */ log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno)); @@ -1470,7 +1489,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) struct snapraid_scan* scan; tommy_node* node; int ret; - int has_persistent_inode; + int has_persistent_inodes; + int has_syncronized_hardlinks; scan = malloc_nofail(sizeof(struct snapraid_scan)); scan->state = state; @@ -1492,16 +1512,19 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) msg_progress("Scanning disk %s...\n", disk->name); /* check if the disk supports persistent inodes */ - ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0); + ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0); if (ret < 0) { /* LCOV_EXCL_START */ log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno)); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - if (!has_persistent_inode) { + if (!has_persistent_inodes) { disk->has_volatile_inodes = 1; } + if (!has_syncronized_hardlinks) { + disk->has_volatile_hardlinks = 1; + } /* if inodes or UUID are not persistent/changed/unsupported */ if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) { diff --git a/cmdline/state.c b/cmdline/state.c index 56ce3454..fefedaae 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1540,7 +1540,7 @@ void state_refresh(struct snapraid_state* state) /* LCOV_EXCL_STOP */ } - ret = fsinfo(disk->dir, 0, &total_space, &free_space); + ret = fsinfo(disk->dir, 0, 0, &total_space, &free_space); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno)); @@ -1568,7 +1568,7 @@ void state_refresh(struct snapraid_state* state) uint64_t free_space; int ret; - ret = fsinfo(state->parity[l].split_map[s].path, 0, &total_space, &free_space); + ret = fsinfo(state->parity[l].split_map[s].path, 0, 0, &total_space, &free_space); if (ret != 0) { /* LCOV_EXCL_START */ log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno)); diff --git a/cmdline/unix.c b/cmdline/unix.c index bb4eeacb..91d9b807 100644 --- a/cmdline/unix.c +++ b/cmdline/unix.c @@ -644,7 +644,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical) return 0; } -int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space) +int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_hardlinks, uint64_t* total_space, uint64_t* free_space) { char type[64]; const char* ptype; @@ -693,11 +693,29 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, u break; } #else - /* by default assume yes */ + /* in Unix inodes are persistent by default */ *has_persistent_inode = 1; #endif } + if (has_syncronized_hardlinks) { +#if HAVE_STATFS && HAVE_STRUCT_STATFS_F_TYPE + switch (st.f_type) { + case 0x5346544E : /* NTFS */ + case 0x4d44 : /* VFAT, "msdos" in the stat command */ + *has_syncronized_hardlinks = 0; + break; + default : + /* by default assume yes */ + *has_syncronized_hardlinks = 1; + break; + } +#else + /* in Unix hardlinks share the same metadata by default */ + *has_syncronized_hardlinks = 1; +#endif + } + if (total_space) { #if HAVE_STATFS *total_space = st.f_bsize * (uint64_t)st.f_blocks;