From f11707400456fb60f40e8084d9196927eb9675a8 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 30 Sep 2014 21:54:49 +0200 Subject: [PATCH] Uses Windows dir stream to make dir listing faster. --- HISTORY | 1 + cmdline/mingw.c | 467 +++++++++++++++++++++++++++++++++--------------- cmdline/mingw.h | 32 ++-- cmdline/scan.c | 44 +++-- configure.ac | 2 +- 5 files changed, 371 insertions(+), 175 deletions(-) diff --git a/HISTORY b/HISTORY index dea9138b..84a06933 100644 --- a/HISTORY +++ b/HISTORY @@ -4,6 +4,7 @@ SnapRAID HISTORY 7.0 2014/9 ========== * Adds AVX2 support to improve parity generation speed. + * Faster directory scanning for Windows. * Prints the time spent waiting for each disk also in 'scrub'. * The CPU usage, speed and ETA estimations are now based on the last 100 seconds rather than from the start. diff --git a/cmdline/mingw.c b/cmdline/mingw.c index 52b3891f..fe1c4826 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -32,7 +32,27 @@ typedef struct _FILE_ATTRIBUTE_TAG_INFO { DWORD FileAttributes; DWORD ReparseTag; } FILE_ATTRIBUTE_TAG_INFO; + +typedef struct _FILE_ID_BOTH_DIR_INFO { + DWORD NextEntryOffset; + DWORD FileIndex; + LARGE_INTEGER CreationTime; + LARGE_INTEGER LastAccessTime; + LARGE_INTEGER LastWriteTime; + LARGE_INTEGER ChangeTime; + LARGE_INTEGER EndOfFile; + LARGE_INTEGER AllocationSize; + DWORD FileAttributes; + DWORD FileNameLength; + DWORD EaSize; + CCHAR ShortNameLength; + WCHAR ShortName[12]; + LARGE_INTEGER FileId; + WCHAR FileName[1]; +} FILE_ID_BOTH_DIR_INFO; + #define FileAttributeTagInfo 9 +#define FileIdBothDirectoryInfo 10 #ifndef IO_REPARSE_TAG_DEDUP #define IO_REPARSE_TAG_DEDUP (0x80000013) @@ -184,26 +204,7 @@ static wchar_t* u8tou16(const char* src) /** * Converts a generic string from UTF16 to UTF8. */ -static char* u16tou8(const wchar_t* src) -{ - int ret; - - if (++conv_utf8 == CONV_ROLL) - conv_utf8 = 0; - - ret = WideCharToMultiByte(CP_UTF8, 0, src, -1, conv_utf8_buffer[conv_utf8], sizeof(conv_utf8_buffer[0]), 0, 0); - if (ret <= 0) { - fwprintf(stderr, L"Error converting name %s from UTF-16 to UTF-8\n", src); - exit(EXIT_FAILURE); - } - - return conv_utf8_buffer[conv_utf8]; -} - -/** - * Converts a generic string from UTF16 to UTF8. - */ -static char* u16tou8n(const wchar_t* src, size_t number_of_wchar, size_t* result_length_without_terminator) +static char* u16tou8(const wchar_t* src, size_t number_of_wchar, size_t* result_length_without_terminator) { int ret; @@ -212,7 +213,7 @@ static char* u16tou8n(const wchar_t* src, size_t number_of_wchar, size_t* result ret = WideCharToMultiByte(CP_UTF8, 0, src, number_of_wchar, conv_utf8_buffer[conv_utf8], sizeof(conv_utf8_buffer[0]), 0, 0); if (ret <= 0) { - fwprintf(stderr, L"Error converting from UTF-16 to UTF-8\n"); + fprintf(stderr, "Error converting from UTF-16 to UTF-8\n"); exit(EXIT_FAILURE); } @@ -340,27 +341,31 @@ static BOOL GetReparseTagInfoByHandle(HANDLE hFile, FILE_ATTRIBUTE_TAG_INFO* lpF } /** - * Converts Windows info to the Unix stat format. + * Converts Windows attr to the Unix stat format. */ -static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ATTRIBUTE_TAG_INFO* tag, struct windows_stat* st) +static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct windows_stat* st) { - uint64_t mtime; - /* Convert special attributes */ - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { + if ((FileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { st->st_mode = S_IFBLK; st->st_desc = "device"; - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* Offline */ + } else if ((FileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* Offline */ st->st_mode = S_IFCHR; st->st_desc = "offline"; - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* Temporary */ + } else if ((FileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* Temporary */ st->st_mode = S_IFCHR; st->st_desc = "temporary"; - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* Reparse point */ - switch (tag->ReparseTag) { - /* For deduplicated files, assume that they are regular ones */ + } else if ((FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* Reparse point */ + switch (ReparseTag) { + /* if we don't have the ReparseTag information */ + case 0 : + /* don't set the st_mode, to set it later calling lstat_ex() */ + st->st_mode = 0; + st->st_desc = "unknown"; + break; + /* for deduplicated files, assume that they are regular ones */ case IO_REPARSE_TAG_DEDUP : - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { + if ((FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { st->st_mode = S_IFDIR; st->st_desc = "directory-dedup"; } else { @@ -369,7 +374,7 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE } break; case IO_REPARSE_TAG_SYMLINK : - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { + if ((FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { st->st_mode = S_IFLNKDIR; st->st_desc = "reparse-point-symlink-dir"; } else { @@ -377,11 +382,11 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE st->st_desc = "reparse-point-symlink-file"; } break; - /* All the other are skipped as reparse-point */ + /* all the other are skipped as reparse-point */ case IO_REPARSE_TAG_MOUNT_POINT : st->st_mode = S_IFCHR; st->st_desc = "reparse-point-mount"; - break; + break; case IO_REPARSE_TAG_NFS : st->st_mode = S_IFCHR; st->st_desc = "reparse-point-nfs"; @@ -391,8 +396,8 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE st->st_desc = "reparse-point"; break; } - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* System */ - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { + } else if ((FileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* System */ + if ((FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { st->st_mode = S_IFCHR; st->st_desc = "system-directory"; } else { @@ -400,7 +405,7 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE st->st_desc = "system-file"; } } else { - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { + if ((FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { st->st_mode = S_IFDIR; st->st_desc = "directory"; } else { @@ -409,8 +414,18 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE } } - /* Store the HIDDEN attribute in a separate field */ - st->st_hidden = (info->dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0; + /* store the HIDDEN attribute in a separate field */ + st->st_hidden = (FileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0; +} + +/** + * Converts Windows info to the Unix stat format. + */ +static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ATTRIBUTE_TAG_INFO* tag, struct windows_stat* st) +{ + uint64_t mtime; + + windows_attr2stat(info->dwFileAttributes, tag->ReparseTag, st); st->st_size = info->nFileSizeHigh; st->st_size <<= 32; @@ -434,7 +449,36 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE st->st_ino <<= 32; st->st_ino |= info->nFileIndexLow; - st->st_nlink = info->nNumberOfLinks; + st->st_dev = info->dwVolumeSerialNumber; +} + +/** + * Converts Windows info to the Unix stat format. + */ +static void windows_stream2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ID_BOTH_DIR_INFO* stream, struct windows_stat* st) +{ + uint64_t mtime; + + /* The FILE_ID_BOTH_DIR_INFO doesn't have the ReparseTag information */ + /* we could use instead FILE_ID_EXTD_DIR_INFO, but it's available only */ + /* from Windows Server 2012 */ + windows_attr2stat(stream->FileAttributes, 0, st); + + st->st_size = stream->EndOfFile.QuadPart; + + mtime = stream->LastWriteTime.QuadPart; + + /* + * Convert to unix time + * + * How To Convert a UNIX time_t to a Win32 FILETIME or SYSTEMTIME + * http://support.microsoft.com/kb/167296 + */ + mtime -= 116444736000000000LL; + st->st_mtime = mtime / 10000000; + st->st_mtimensec = (mtime % 10000000) * 100; + + st->st_ino = stream->FileId.QuadPart; st->st_dev = info->dwVolumeSerialNumber; } @@ -446,71 +490,7 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s { uint64_t mtime; - /* Convert special attributes */ - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { - st->st_mode = S_IFBLK; - st->st_desc = "device"; - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* Offline */ - st->st_mode = S_IFCHR; - st->st_desc = "offline"; - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* Temporary */ - st->st_mode = S_IFCHR; - st->st_desc = "temporary"; - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* Reparse point */ - switch (info->dwReserved0) { - /* For deduplicated files, assume that they are regular ones */ - case IO_REPARSE_TAG_DEDUP : - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { - st->st_mode = S_IFDIR; - st->st_desc = "directory-dedup"; - } else { - st->st_mode = S_IFREG; - st->st_desc = "regular-dedup"; - } - break; - case IO_REPARSE_TAG_SYMLINK : - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { - st->st_mode = S_IFLNKDIR; - st->st_desc = "reparse-point-symlink-dir"; - } else { - st->st_mode = S_IFLNK; - st->st_desc = "reparse-point-symlink-file"; - } - break; - /* All the other are skipped as reparse-point */ - case IO_REPARSE_TAG_MOUNT_POINT : - st->st_mode = S_IFCHR; - st->st_desc = "reparse-point-mount"; - break; - case IO_REPARSE_TAG_NFS : - st->st_mode = S_IFCHR; - st->st_desc = "reparse-point-nfs"; - break; - default : - st->st_mode = S_IFCHR; - st->st_desc = "reparse-point"; - break; - } - } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* System */ - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { - st->st_mode = S_IFCHR; - st->st_desc = "system-directory"; - } else { - st->st_mode = S_IFREG; - st->st_desc = "system-file"; - } - } else { - if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { - st->st_mode = S_IFDIR; - st->st_desc = "directory"; - } else { - st->st_mode = S_IFREG; - st->st_desc = "regular"; - } - } - - /* Store the HIDDEN attribute in a separate field */ - st->st_hidden = (info->dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0; + windows_attr2stat(info->dwFileAttributes, info->dwReserved0, st); st->st_size = info->nFileSizeHigh; st->st_size <<= 32; @@ -533,9 +513,6 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s /* No inode information available */ st->st_ino = 0; - /* No link information available */ - st->st_nlink = 0; - /* No device information available */ st->st_dev = 0; } @@ -545,19 +522,35 @@ static void windows_finddata2dirent(const WIN32_FIND_DATAW* info, struct windows const char* name; size_t len; - name = u16tou8(info->cFileName); - - len = strlen(name); + name = u16tou8(info->cFileName, wcslen(info->cFileName), &len); if (len + 1 >= sizeof(dirent->d_name)) { fprintf(stderr, "Name too long\n"); exit(EXIT_FAILURE); } - memcpy(dirent->d_name, name, len + 1); + memcpy(dirent->d_name, name, len); + dirent->d_name[len] = 0; - /* Store the HIDDEN attribute in a separate field */ - dirent->d_hidden = (info->dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0; + windows_finddata2stat(info, &dirent->d_stat); +} + +static void windows_stream2dirent(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ID_BOTH_DIR_INFO* stream, struct windows_dirent* dirent) +{ + const char* name; + size_t len; + + name = u16tou8(stream->FileName, stream->FileNameLength / 2, &len); + + if (len + 1 >= sizeof(dirent->d_name)) { + fprintf(stderr, "Name too long\n"); + exit(EXIT_FAILURE); + } + + memcpy(dirent->d_name, name, len); + dirent->d_name[len] = 0; + + windows_stream2stat(info, stream, &dirent->d_stat); } /** @@ -658,7 +651,7 @@ int windows_lstat(const char* file, struct windows_stat* st) void windows_dirent_lstat(const struct windows_dirent* dd, struct windows_stat* st) { - windows_finddata2stat(&dd->d_data, st); + memcpy(st, &dd->d_stat, sizeof(struct windows_stat)); } int windows_mkdir(const char* file) @@ -931,7 +924,22 @@ int windows_open(const char* file, int flags, ...) return ret; } -windows_dir* windows_opendir(const char* dir) +struct windows_dir_struct { + BY_HANDLE_FILE_INFORMATION info; + WIN32_FIND_DATAW find; + HANDLE h; + struct windows_dirent entry; + unsigned char* buffer; + unsigned buffer_size; + unsigned buffer_pos; + int state; +}; + +#define DIR_STATE_EOF -1 /**< End of the dir stream */ +#define DIR_STATE_EMPTY 0 /**< The entry is empty. */ +#define DIR_STATE_FILLED 1 /**< The entry is valid. */ + +static windows_dir* windows_opendir_find(const char* dir) { wchar_t* wdir; windows_dir* dirstream; @@ -952,12 +960,12 @@ windows_dir* windows_opendir(const char* dir) wdir[len++] = L'*'; wdir[len++] = 0; - dirstream->h = FindFirstFileW(wdir, &dirstream->buffer.d_data); + dirstream->h = FindFirstFileW(wdir, &dirstream->find); if (dirstream->h == INVALID_HANDLE_VALUE) { DWORD error = GetLastError(); if (error == ERROR_FILE_NOT_FOUND) { - dirstream->flags = -1; /* empty dir */ + dirstream->state = DIR_STATE_EOF; return dirstream; } @@ -966,47 +974,48 @@ windows_dir* windows_opendir(const char* dir) return 0; } - dirstream->flags = 1; - - windows_finddata2dirent(&dirstream->buffer.d_data, &dirstream->buffer); + windows_finddata2dirent(&dirstream->find, &dirstream->entry); + dirstream->state = DIR_STATE_FILLED; return dirstream; } -struct windows_dirent* windows_readdir(windows_dir* dirstream) +static struct windows_dirent* windows_readdir_find(windows_dir* dirstream) { - if (dirstream->flags == -1) { - errno = 0; /* end of stream */ - return 0; - } + if (dirstream->state == DIR_STATE_EMPTY) { + if (!FindNextFileW(dirstream->h, &dirstream->find)) { + DWORD error = GetLastError(); - if (dirstream->flags == 1) { - dirstream->flags = 0; - return &dirstream->buffer; - } + if (error != ERROR_NO_MORE_FILES) { + windows_errno(error); + return 0; + } - if (!FindNextFileW(dirstream->h, &dirstream->buffer.d_data)) { - DWORD error = GetLastError(); - - if (error == ERROR_NO_MORE_FILES) { - errno = 0; /* end of stream */ - return 0; + dirstream->state = DIR_STATE_EOF; + } else { + windows_finddata2dirent(&dirstream->find, &dirstream->entry); + dirstream->state = DIR_STATE_FILLED; } - - windows_errno(error); - return 0; } - windows_finddata2dirent(&dirstream->buffer.d_data, &dirstream->buffer); + if (dirstream->state == DIR_STATE_FILLED) { + dirstream->state = DIR_STATE_EMPTY; + return &dirstream->entry; + } - return &dirstream->buffer; + /* otherwise it's the end of stream */ + assert(dirstream->state == DIR_STATE_EOF); + errno = 0; + + return 0; } -int windows_closedir(windows_dir* dirstream) +static int windows_closedir_find(windows_dir* dirstream) { if (dirstream->h != INVALID_HANDLE_VALUE) { if (!FindClose(dirstream->h)) { DWORD error = GetLastError(); + free(dirstream); windows_errno(error); @@ -1019,9 +1028,177 @@ int windows_closedir(windows_dir* dirstream) return 0; } +static int windows_first_stream(windows_dir* dirstream) +{ + FILE_ID_BOTH_DIR_INFO* fd; + + if (!ptr_GetFileInformationByHandleEx(dirstream->h, FileIdBothDirectoryInfo, dirstream->buffer, dirstream->buffer_size)) { + DWORD error = GetLastError(); + + if (error == ERROR_NO_MORE_FILES) { + dirstream->state = DIR_STATE_EOF; + return 0; + } + + windows_errno(error); + return -1; + } + + /* get the first entry */ + dirstream->buffer_pos = 0; + fd = (FILE_ID_BOTH_DIR_INFO*)dirstream->buffer; + windows_stream2dirent(&dirstream->info, fd, &dirstream->entry); + dirstream->state = DIR_STATE_FILLED; + + return 0; +} + +static int windows_next_stream(windows_dir* dirstream) +{ + FILE_ID_BOTH_DIR_INFO* fd; + + /* last entry read */ + fd = (FILE_ID_BOTH_DIR_INFO*)(dirstream->buffer + dirstream->buffer_pos); + + /* check if there is a next one */ + if (fd->NextEntryOffset == 0) { + /* if not, fill it up again */ + if (windows_first_stream(dirstream) != 0) + return -1; + return 0; + } + + /* go to the next one */ + dirstream->buffer_pos += fd->NextEntryOffset; + fd = (FILE_ID_BOTH_DIR_INFO*)(dirstream->buffer + dirstream->buffer_pos); + windows_stream2dirent(&dirstream->info, fd, &dirstream->entry); + dirstream->state = DIR_STATE_FILLED; + + return 0; +} + +static windows_dir* windows_opendir_stream(const char* dir) +{ + windows_dir* dirstream; + WCHAR* wdir; + + dirstream = malloc(sizeof(windows_dir)); + if (!dirstream) { + fprintf(stderr, "Low memory\n"); + exit(EXIT_FAILURE); + } + + wdir = convert(dir); + + /* uses a 64 kB buffer for reading directory */ + dirstream->buffer_size = 64 * 1024; + dirstream->buffer = malloc(dirstream->buffer_size); + if (!dirstream->buffer) { + fprintf(stderr, "Low memory\n"); + exit(EXIT_FAILURE); + } + + dirstream->h = CreateFileW(wdir, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, 0); + if (dirstream->h == INVALID_HANDLE_VALUE) { + DWORD error = GetLastError(); + free(dirstream->buffer); + free(dirstream); + windows_errno(error); + return 0; + } + + /* get dir information for the VolumeSerialNumber */ + /* this value is used for all the files in the dir */ + if (!GetFileInformationByHandle(dirstream->h, &dirstream->info)) { + DWORD error = GetLastError(); + CloseHandle(dirstream->h); + free(dirstream->buffer); + free(dirstream); + windows_errno(error); + return 0; + } + + if (windows_first_stream(dirstream) != 0) { + CloseHandle(dirstream->h); + free(dirstream->buffer); + free(dirstream); + return 0; + } + + return dirstream; +} + +static struct windows_dirent* windows_readdir_stream(windows_dir* dirstream) +{ + if (dirstream->state == DIR_STATE_EMPTY) { + if (windows_next_stream(dirstream) != 0) { + free(dirstream->buffer); + free(dirstream); + return 0; + } + } + + if (dirstream->state == DIR_STATE_FILLED) { + dirstream->state = DIR_STATE_EMPTY; + return &dirstream->entry; + } + + /* otherwise it's the end of stream */ + assert(dirstream->state == DIR_STATE_EOF); + errno = 0; + + return 0; +} + +static int windows_closedir_stream(windows_dir* dirstream) +{ + if (dirstream->h != INVALID_HANDLE_VALUE) { + if (!CloseHandle(dirstream->h)) { + DWORD error = GetLastError(); + + free(dirstream->buffer); + free(dirstream); + + windows_errno(error); + return -1; + } + } + + free(dirstream->buffer); + free(dirstream); + + return 0; +} + +windows_dir* windows_opendir(const char* dir) +{ + /* if we have GetFileInformationByHandleEx() we can read */ + /* the directory using a stream */ + if (ptr_GetFileInformationByHandleEx != 0) + return windows_opendir_stream(dir); + else + return windows_opendir_find(dir); +} + +struct windows_dirent* windows_readdir(windows_dir* dirstream) +{ + if (ptr_GetFileInformationByHandleEx != 0) + return windows_readdir_stream(dirstream); + else + return windows_readdir_find(dirstream); +} + +int windows_closedir(windows_dir* dirstream) +{ + if (ptr_GetFileInformationByHandleEx != 0) + return windows_closedir_stream(dirstream); + else + return windows_closedir_find(dirstream); +} + int windows_dirent_hidden(struct dirent* dd) { - return dd->d_hidden; + return dd->d_stat.st_hidden; } const char* windows_stat_desc(struct stat* st) @@ -1134,7 +1311,7 @@ int windows_readlink(const char* file, char* buffer, size_t size) } /* convert the name to UTF-8 */ - name = u16tou8n(rdb->SymbolicLinkReparseBuffer.PathBuffer + rdb->SymbolicLinkReparseBuffer.PrintNameOffset, + name = u16tou8(rdb->SymbolicLinkReparseBuffer.PathBuffer + rdb->SymbolicLinkReparseBuffer.PrintNameOffset, rdb->SymbolicLinkReparseBuffer.PrintNameLength / 2, &len); /* check for overflow */ diff --git a/cmdline/mingw.h b/cmdline/mingw.h index 0b2a5dc2..9d660c85 100644 --- a/cmdline/mingw.h +++ b/cmdline/mingw.h @@ -80,6 +80,7 @@ #define dirent_hidden windows_dirent_hidden #define HAVE_STRUCT_DIRENT_D_STAT 1 #undef HAVE_STRUCT_DIRENT_D_INO +#undef HAVE_STRUCT_STAT_ST_NLINK #define dirent_lstat windows_dirent_lstat #define stat_desc windows_stat_desc #undef sleep @@ -130,21 +131,22 @@ struct windows_stat { int64_t st_mtime; int32_t st_mtimensec; uint32_t st_mode; - uint32_t st_nlink; uint32_t st_dev; int st_hidden; const char* st_desc; }; /** - * Like the C fstat() including the inode/device information. + * Like the C fstat(). */ int windows_fstat(int fd, struct windows_stat* st); /** - * Like the C lstat() but without the inode information. - * In Windows the inode information is not reported. - * In Windows and in case of hardlinks, the size and the attributes of the file can + * Like the C lstat() but without some information. + * + * The st_ino field may be 0 if it's not possible to read it in a fast way. + * + * In case of hardlinks, the size and the attributes of the file can * be completely bogus, because changes made by other hardlinks are reported in the * directory entry only when the file is opened. * @@ -164,7 +166,7 @@ int windows_fstat(int fd, struct windows_stat* st); int windows_lstat(const char* file, struct windows_stat* st); /** - * Like the C stat() including the inode/device information. + * Like the C stat(). */ int windows_stat(const char* file, struct windows_stat* st); @@ -179,7 +181,8 @@ int windows_mkdir(const char* file); int windows_rmdir(const char* file); /** - * Like the C lstat() including the inode/device information. + * Like the C lstat(). + * It doesn't work for all kind of files and directories. For example "\System Volume Information" cannot be opened. * Note that instead lstat() works for all the files. */ @@ -238,18 +241,13 @@ int windows_open(const char* file, int flags, ...); */ struct windows_dirent { char d_name[PATH_MAX]; - int d_hidden; - WIN32_FIND_DATAW d_data; + struct windows_stat d_stat; }; /** * Like the C DIR. */ -struct windows_dir_struct { - HANDLE h; - struct windows_dirent buffer; - int flags; -}; +struct windows_dir_struct; typedef struct windows_dir_struct windows_dir; /** @@ -269,7 +267,11 @@ int windows_closedir(windows_dir* dirstream); /** * Convert a dirent record to a lstat record. - * Just like the one obtained calling lstat(). + * + * The st_mode field may be 0 if the file is a reparse point. + * The st_ino field may be 0 if it's not possible to read it in a fast way. + * + * In such cases, call lstat_ex() to fill the missing fields. */ void windows_dirent_lstat(const struct windows_dirent* dd, struct windows_stat* st); diff --git a/cmdline/scan.c b/cmdline/scan.c index e00c17a6..0526e2a6 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -499,16 +499,17 @@ static void scan_file(struct snapraid_scan* scan, int output, const char* sub, c ) { /* check if multiple files have the same inode */ if (file_flag_has(file, FILE_IS_PRESENT)) { - if (st->st_nlink > 1) { - /* it's a hardlink */ - scan_link(scan, output, sub, file->sub, FILE_IS_HARDLINK); - return; - } else { +#if HAVE_STRUCT_STAT_ST_NLINK + if (st->st_nlink <= 1) { /* LCOV_EXCL_START */ fprintf(stderr, "Internal inode '%" PRIu64 "' inconsistency for file '%s%s' already present\n", (uint64_t)st->st_ino, disk->dir, sub); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } +#endif + /* it's a hardlink */ + scan_link(scan, output, sub, file->sub, FILE_IS_HARDLINK); + return; } /* mark as present */ @@ -1083,6 +1084,18 @@ static int scan_dir(struct snapraid_scan* scan, int output, const char* dir, con /* get the type from stat */ st = DSTAT(path_next, dd, &st_buf); +#if HAVE_LSTAT_EX + /* if the stat entry is incomplete, take care to fill it using the extended lstat() */ + if (st->st_mode == 0) { + if (lstat_ex(path_next, st) != 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } +#endif + if (S_ISREG(st->st_mode)) type = 0; else if (S_ISLNK(st->st_mode)) @@ -1097,19 +1110,22 @@ static int scan_dir(struct snapraid_scan* scan, int output, const char* dir, con if (filter_path(&state->filterlist, disk->name, sub_next) == 0) { uint64_t physical; - /* late stat */ - if (!st) st = DSTAT(path_next, dd, &st_buf); + /* late stat, if not yet called */ + if (!st) + st = DSTAT(path_next, dd, &st_buf); #if HAVE_LSTAT_EX - /* get inode info about the file, Windows needs an additional step */ - /* also for hardlink, the real size of the file is read here */ - if (lstat_ex(path_next, st) != 0) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Error in stat_inode file '%s'. %s.\n", path_next, strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + /* if the stat entry is incomplete, take care to fill it using the extended lstat() */ + if (st->st_ino == 0) { + if (lstat_ex(path_next, st) != 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } #endif + if (state->opt.force_order == SORT_PHYSICAL) { if (filephy(path_next, st, &physical) != 0) { /* LCOV_EXCL_START */ diff --git a/configure.ac b/configure.ac index cf4b2e39..3224d3bc 100644 --- a/configure.ac +++ b/configure.ac @@ -42,7 +42,7 @@ AC_TYPE_UINT8_T AC_TYPE_INT8_T AC_STRUCT_DIRENT_D_INO AC_STRUCT_DIRENT_D_TYPE -AC_CHECK_MEMBERS([struct stat.st_mtim.tv_nsec, struct stat.st_mtimensec, struct stat.st_mtimespec.tv_nsec], [], [], [[ +AC_CHECK_MEMBERS([struct stat.st_nlink, struct stat.st_mtim.tv_nsec, struct stat.st_mtimensec, struct stat.st_mtimespec.tv_nsec], [], [], [[ #if HAVE_SYS_TYPES_H #include #endif