rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Apply new hardlink behaviour also to NTFS in nix platforms

Browse source 
This is a specific NTFS behaviour, not limited to Windows.
This commit is contained in:
Andrea Mazzoleni 2017-10-29 18:07:01 +01:00
parent 58a7375661
commit f778fb691f
9 changed files with 88 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

8
HISTORY
View file

@ -1,12 +1,12 @@
SnapRAID HISTORY
================
11.2 2017/10
11.2 2017/11
============
* Fix recognition of Windows hardlinks. They behave differently than
Unix hardlinks and this could result in SnapRAID reporting
* Fix recognition of NTFS hardlinks. They behave differently than
standard Unix hardlinks and this could result in SnapRAID reporting
internal incosistency errors for detecting links to the same file
with different attributes.
with different metadata attributes.
* More efficient 'pool' command that updates only the links
that need to be updated. This ensures that no change is
done, avoiding to trigger a directory rescan of other programs.

1
cmdline/elem.c
View file

@ -760,6 +760,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
disk->free_blocks = 0;
disk->first_free_block = 0;
disk->has_volatile_inodes = 0;
disk->has_volatile_hardlinks = 0;
disk->has_unreliable_physical = 0;
disk->has_different_uuid = 0;
disk->has_unsupported_uuid = *uuid == 0; /* empty UUID means unsupported */

1
cmdline/elem.h
View file

@ -348,6 +348,7 @@ struct snapraid_disk {
block_off_t first_free_block;
int has_volatile_inodes; /**< If the underline file-system has not persistent inodes. */
int has_volatile_hardlinks; /**< If the underline file-system has not syncronized metadata for hardlink (NTFS). */
int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same file-system. */
int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */

13
cmdline/mingw.c
View file

@ -501,6 +501,8 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_
st->st_ino <<= 32;
st->st_ino |= info->nFileIndexLow;
st->st_nlink = info->nNumberOfLinks;
st->st_dev = info->dwVolumeSerialNumber;
/* GetFileInformationByHandle() ensures to return synced information */
@ -553,6 +555,8 @@ static int windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FIL
st->st_ino = stream->FileId.QuadPart;
st->st_nlink = info->nNumberOfLinks;
st->st_dev = info->dwVolumeSerialNumber;
/* directory listing doesn't ensure to return synced information */
@ -598,6 +602,9 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s
/* No inode information available */
st->st_ino = 0;
/* No link information available */
st->st_nlink = 0;
/* No device information available */
st->st_dev = 0;
@ -1705,7 +1712,7 @@ int filephy(const char* file, uint64_t size, uint64_t* physical)
return 0;
}
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space)
int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_hardlinks, uint64_t* total_space, uint64_t* free_space)
{
wchar_t conv_buf[CONV_MAX];
@ -1713,6 +1720,10 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, u
if (has_persistent_inode)
*has_persistent_inode = 1;
/* NTFS doesn't syncronize hardlinks metadata */
if (has_syncronized_hardlinks)
*has_syncronized_hardlinks = 0;
if (free_space || total_space) {
ULARGE_INTEGER total_bytes;
ULARGE_INTEGER total_free_bytes;

3
cmdline/mingw.h
View file

@ -94,7 +94,7 @@
#define dirent_hidden windows_dirent_hidden
#define HAVE_STRUCT_DIRENT_D_STAT 1
#undef HAVE_STRUCT_DIRENT_D_INO
#undef HAVE_STRUCT_STAT_ST_NLINK
#define HAVE_STRUCT_STAT_ST_NLINK 1
#define dirent_lstat windows_dirent_lstat
#define stat_desc windows_stat_desc
#undef sleep
@ -148,6 +148,7 @@ struct windows_stat {
int32_t st_mtimensec;
uint32_t st_mode;
uint32_t st_dev;
uint32_t st_nlink;
int st_hidden;
const char* st_desc;
int st_sync; /**< If the information are in sync with the file-system. */

2
cmdline/portable.h
View file

@ -345,7 +345,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical);
* Check if the underline file-system support persistent inodes.
* Return -1 on error, 0 on success.
*/
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space);
int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_hardlinks, uint64_t* total_space, uint64_t* free_space);
/**
* Get the tick counter value.

67
cmdline/scan.c
View file

@ -501,11 +501,16 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
* because the metadata in the directory is updated only when the file
* is closed.
*
* The same happens for hardlinks that duplicate metatada.
* The link metadata is updated only when the link is opened.
* This extends also to st_size and st_nlink.
*
* See also:
* Why is the file size reported incorrectly for files that are still being written to?
* http://blogs.msdn.com/b/oldnewthing/archive/2011/12/26/10251026.aspx
*/
log_fatal("WARNING! Detected uncached time change for file '%s'\n", sub);
log_fatal("WARNING! Detected uncached time change from %" PRIu64 ".%09u to %" PRIu64 ".%09u for file '%s'\n",
(uint64_t)st->st_mtime, (uint32_t)st->st_mtimensec, (uint64_t)synced_st.st_mtime, (uint32_t)synced_st.st_mtimensec, sub);
log_fatal("It's better if you run SnapRAID without other processes running.\n");
#endif
st->st_mtime = synced_st.st_mtime;
@ -514,15 +519,26 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc
if (st->st_size != synced_st.st_size) {
#ifndef _WIN32
log_fatal("WARNING! Detected uncached size change for file '%s'\n", sub);
log_fatal("WARNING! Detected uncached size change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
(uint64_t)st->st_size, (uint64_t)synced_st.st_size, sub);
log_fatal("It's better if you run SnapRAID without other processes running.\n");
#endif
st->st_size = synced_st.st_size;
}
if (st->st_nlink != synced_st.st_nlink) {
#ifndef _WIN32
log_fatal("WARNING! Detected uncached nlink change from %u to %u for file '%s'\n",
(uint32_t)st->st_nlink , (uint32_t)synced_st.st_nlink, sub);
log_fatal("It's better if you run SnapRAID without other processes running.\n");
#endif
st->st_nlink = synced_st.st_nlink;
}
if (st->st_ino != synced_st.st_ino) {
log_fatal("DANGER! Detected uncached inode change for file '%s'\n", sub);
log_fatal("Please ensure to run SnapRAID without other processes running.\n");
log_fatal("DANGER! Detected uncached inode change from %" PRIu64 " to %" PRIu64 " for file '%s'\n",
(uint64_t)st->st_ino, (uint64_t)synced_st.st_ino, sub);
log_fatal("It's better if you run SnapRAID without other processes running.\n");
/* at this point, it's too late to change inode */
/* and having inconsistent inodes may result to internal failures */
/* so, it's better to abort */
@ -666,14 +682,14 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
) {
/* check if multiple files have the same inode */
if (file_flag_has(file, FILE_IS_PRESENT)) {
#if HAVE_STRUCT_STAT_ST_NLINK
if (st->st_nlink <= 1) {
/* if has_volatile_hardlinks is true, the nlink value is not reliable */
if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
/* LCOV_EXCL_START */
log_fatal("Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub);
os_abort();
/* LCOV_EXCL_STOP */
}
#endif
/* it's a hardlink */
scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
return;
@ -749,20 +765,23 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub,
* ...both time and size of A and B don't match!
*/
if (file_flag_has(file, FILE_IS_PRESENT)) {
#ifdef _WIN32
/* "suppose" a hardlink */
/* if has_volatile_hardlinks is true, the nlink value is not reliable */
if (!disk->has_volatile_hardlinks && st->st_nlink == 1) {
/* LCOV_EXCL_START */
log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different attributes: size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d\n",
file->inode, disk->dir, sub, disk->dir, file->sub,
file->size, (uint64_t)st->st_size,
file->mtime_sec, (uint64_t)st->st_mtime,
file->mtime_nsec, STAT_NSEC(st));
os_abort();
/* LCOV_EXCL_STOP */
}
/* LCOV_EXCL_START */
/* suppose it's hardlink with not synced metadata */
scan_link(scan, is_diff, sub, file->sub, FILE_IS_HARDLINK);
return;
#else
/* LCOV_EXCL_START */
log_fatal("Internal inode '%" PRIu64 "' inconsistency for files '%s%s' and '%s%s' with same inode but different (size %" PRIu64 "?%" PRIu64 ", sec %" PRIu64 "?%" PRIu64 ", nsec %d?%d)\n",
file->inode, disk->dir, sub, disk->dir, file->sub,
file->size, (uint64_t)st->st_size,
file->mtime_sec, (uint64_t)st->st_mtime,
file->mtime_nsec, STAT_NSEC(st));
os_abort();
/* LCOV_EXCL_STOP */
#endif
}
/* assume a previously used inode, it's the worst case */
@ -1351,7 +1370,7 @@ static int scan_dir(struct snapraid_scan* scan, int level, int is_diff, const ch
#if HAVE_LSTAT_SYNC
/* if the st_ino field is missing, takes care to fill it using the extended lstat() */
/* this can happen only in Windows */
if (st->st_ino == 0) {
if (st->st_ino == 0 || st->st_nlink == 0) {
if (lstat_sync(path_next, st, 0) != 0) {
/* LCOV_EXCL_START */
log_fatal("Error in stat file '%s'. %s.\n", path_next, strerror(errno));
@ -1470,7 +1489,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
struct snapraid_scan* scan;
tommy_node* node;
int ret;
int has_persistent_inode;
int has_persistent_inodes;
int has_syncronized_hardlinks;
scan = malloc_nofail(sizeof(struct snapraid_scan));
scan->state = state;
@ -1492,16 +1512,19 @@ static int state_diffscan(struct snapraid_state* state, int is_diff)
msg_progress("Scanning disk %s...\n", disk->name);
/* check if the disk supports persistent inodes */
ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0);
ret = fsinfo(disk->dir, &has_persistent_inodes, &has_syncronized_hardlinks, 0, 0);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!has_persistent_inode) {
if (!has_persistent_inodes) {
disk->has_volatile_inodes = 1;
}
if (!has_syncronized_hardlinks) {
disk->has_volatile_hardlinks = 1;
}
/* if inodes or UUID are not persistent/changed/unsupported */
if (disk->has_volatile_inodes || disk->has_different_uuid || disk->has_unsupported_uuid) {

4
cmdline/state.c
View file

@ -1540,7 +1540,7 @@ void state_refresh(struct snapraid_state* state)
/* LCOV_EXCL_STOP */
}
ret = fsinfo(disk->dir, 0, &total_space, &free_space);
ret = fsinfo(disk->dir, 0, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing disk '%s' to get file-system info. %s.\n", disk->dir, strerror(errno));
@ -1568,7 +1568,7 @@ void state_refresh(struct snapraid_state* state)
uint64_t free_space;
int ret;
ret = fsinfo(state->parity[l].split_map[s].path, 0, &total_space, &free_space);
ret = fsinfo(state->parity[l].split_map[s].path, 0, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));

22
cmdline/unix.c
View file

@ -644,7 +644,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical)
return 0;
}
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space)
int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_hardlinks, uint64_t* total_space, uint64_t* free_space)
{
char type[64];
const char* ptype;
@ -693,11 +693,29 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, u
break;
}
#else
/* by default assume yes */
/* in Unix inodes are persistent by default */
*has_persistent_inode = 1;
#endif
}
if (has_syncronized_hardlinks) {
#if HAVE_STATFS && HAVE_STRUCT_STATFS_F_TYPE
switch (st.f_type) {
case 0x5346544E : /* NTFS */
case 0x4d44 : /* VFAT, "msdos" in the stat command */
*has_syncronized_hardlinks = 0;
break;
default :
/* by default assume yes */
*has_syncronized_hardlinks = 1;
break;
}
#else
/* in Unix hardlinks share the same metadata by default */
*has_syncronized_hardlinks = 1;
#endif
}
if (total_space) {
#if HAVE_STATFS
*total_space = st.f_bsize * (uint64_t)st.f_blocks;