From 55006d76f3769bb5eabbd19c0922380ac092ca55 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 14 Oct 2014 22:47:21 +0200 Subject: [PATCH] Merges lstat_sync() and filephy() in Windows reduce syscalls. --- TODO | 6 --- cmdline/mingw.c | 110 ++++++++++++++++++++++++++---------------------- cmdline/mingw.h | 5 ++- cmdline/scan.c | 26 +++++++++--- 4 files changed, 83 insertions(+), 64 deletions(-) diff --git a/TODO b/TODO index 1ba3f05d..25c97af8 100644 --- a/TODO +++ b/TODO @@ -13,12 +13,6 @@ get misinterpreted. * Shall CreateFileW use FILE_SHARE_DELETE ? -* Merge lstat_sync() and filephy() ? This could save one CreateFileW() -but it's not clear how it will behave with ReparsePoint, like dedup files. -lsync_stat opens with BACKUP_SEMANTICS/REPARSE_POINT then instead filephy() -opens with 0. -See merge_lstat_sync_and_filephy.diff in contrib/ - - Minor * Don't insert new files if they are too recent and opened by other diff --git a/cmdline/mingw.c b/cmdline/mingw.c index fd53db34..41d8529e 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -690,7 +690,56 @@ int windows_rmdir(const char* file) return 0; } -int lstat_sync(const char* file, struct windows_stat* st) +static BOOL GetFilePhysicalOffset(HANDLE h, uint64_t* physical) +{ + STARTING_VCN_INPUT_BUFFER svib; + unsigned char rpb_buffer[sizeof(RETRIEVAL_POINTERS_BUFFER)]; + RETRIEVAL_POINTERS_BUFFER* rpb = (RETRIEVAL_POINTERS_BUFFER*)&rpb_buffer; + BOOL ret; + DWORD n; + + /* in Wine FSCTL_GET_RETRIVIAL_POINTERS is not supported */ + if (is_wine) { + *physical = FILEPHY_UNREPORTED_OFFSET; + return TRUE; + } + + /* set the output variable, just to be safe */ + rpb->ExtentCount = 0; + + /* read the physical address */ + svib.StartingVcn.QuadPart = 0; + ret = DeviceIoControl(h, FSCTL_GET_RETRIEVAL_POINTERS, &svib, sizeof(svib), rpb_buffer, sizeof(rpb_buffer), &n, 0); + if (!ret) { + DWORD error = GetLastError(); + if (error == ERROR_MORE_DATA) { + /* we ignore ERROR_MODE_DATA because we are interested only at the first entry */ + /* and this is the expected error if the files has more entries */ + } else if (error == ERROR_HANDLE_EOF) { + /* if the file is small, it can be stored in the Master File Table (MFT) */ + /* and then it doesn't have a physical address */ + /* In such case we report a specific fake address, to report this special condition */ + /* that it's different from the 0 offset reported by the underline file system */ + *physical = FILEPHY_WITHOUT_OFFSET; + return TRUE; + } else if (error == ERROR_NOT_SUPPORTED) { + /* for disks shared on network this operation is not supported */ + *physical = FILEPHY_UNREPORTED_OFFSET; + return TRUE; + } else { + return FALSE; + } + } + + if (rpb->ExtentCount < 1) + *physical = FILEPHY_UNREPORTED_OFFSET; + else + *physical = rpb->Extents[0].Lcn.QuadPart + FILEPHY_REAL_OFFSET; + + return TRUE; +} + +int lstat_sync(const char* file, struct windows_stat* st, uint64_t* physical) { BY_HANDLE_FILE_INFORMATION info; FILE_ATTRIBUTE_TAG_INFO tag; @@ -722,6 +771,16 @@ int lstat_sync(const char* file, struct windows_stat* st) return -1; } + /* read the physical offset, only if a pointer is provided */ + if (physical != 0) { + if (!GetFilePhysicalOffset(h, physical)) { + DWORD error = GetLastError(); + CloseHandle(h); + windows_errno(error); + return -1; + } + } + if (!CloseHandle(h)) { windows_errno(GetLastError()); return -1; @@ -1348,55 +1407,6 @@ int devuuid(uint64_t device, char* uuid, size_t uuid_size) return 0; } -static BOOL GetFilePhysicalOffset(HANDLE h, uint64_t* physical) -{ - STARTING_VCN_INPUT_BUFFER svib; - unsigned char rpb_buffer[sizeof(RETRIEVAL_POINTERS_BUFFER)]; - RETRIEVAL_POINTERS_BUFFER* rpb = (RETRIEVAL_POINTERS_BUFFER*)&rpb_buffer; - BOOL ret; - DWORD n; - - /* in Wine FSCTL_GET_RETRIVIAL_POINTERS is not supported */ - if (is_wine) { - *physical = FILEPHY_UNREPORTED_OFFSET; - return TRUE; - } - - /* set the output variable, just to be safe */ - rpb->ExtentCount = 0; - - /* read the physical address */ - svib.StartingVcn.QuadPart = 0; - ret = DeviceIoControl(h, FSCTL_GET_RETRIEVAL_POINTERS, &svib, sizeof(svib), rpb_buffer, sizeof(rpb_buffer), &n, 0); - if (!ret) { - DWORD error = GetLastError(); - if (error == ERROR_MORE_DATA) { - /* we ignore ERROR_MODE_DATA because we are interested only at the first entry */ - /* and this is the expected error if the files has more entries */ - } else if (error == ERROR_HANDLE_EOF) { - /* if the file is small, it can be stored in the Master File Table (MFT) */ - /* and then it doesn't have a physical address */ - /* In such case we report a specific fake address, to report this special condition */ - /* that it's different from the 0 offset reported by the underline file system */ - *physical = FILEPHY_WITHOUT_OFFSET; - return TRUE; - } else if (error == ERROR_NOT_SUPPORTED) { - /* for disks shared on network this operation is not supported */ - *physical = FILEPHY_UNREPORTED_OFFSET; - return TRUE; - } else { - return FALSE; - } - } - - if (rpb->ExtentCount < 1) - *physical = FILEPHY_UNREPORTED_OFFSET; - else - *physical = rpb->Extents[0].Lcn.QuadPart + FILEPHY_REAL_OFFSET; - - return TRUE; -} - int filephy(const char* file, uint64_t size, uint64_t* physical) { HANDLE h; diff --git a/cmdline/mingw.h b/cmdline/mingw.h index 88a92b42..5d247d5a 100644 --- a/cmdline/mingw.h +++ b/cmdline/mingw.h @@ -185,7 +185,8 @@ int windows_rmdir(const char* file); /** * Like the C lstat(), but with some limitations. * - * This call fills all the st_* fields of the stat struct. + * This call fills all the st_* fields of the stat struct, + * and if provided the pointer, also the physical offset. * * It doesn't work for all kinds of files and directories. * You must call it only for regular files. @@ -194,7 +195,7 @@ int windows_rmdir(const char* file); * Note that instead lstat() works for all the files. */ #define HAVE_LSTAT_SYNC 1 -int lstat_sync(const char* file, struct windows_stat* st); +int lstat_sync(const char* file, struct windows_stat* st, uint64_t* physical); /** * Like the C ftruncate(). diff --git a/cmdline/scan.c b/cmdline/scan.c index c63d9bab..495cd150 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -295,8 +295,10 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid struct snapraid_state* state = scan->state; struct snapraid_disk* disk = scan->disk; - /* if we sort for physical offsets we have to read them for the new files */ - if (state->opt.force_order == SORT_PHYSICAL) { + /* if we sort for physical offsets we have to read them for new files */ + if (state->opt.force_order == SORT_PHYSICAL + && file->physical == FILEPHY_UNREAD_OFFSET + ) { char path_next[PATH_MAX]; pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, file->sub); @@ -318,10 +320,13 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid * This is needed by Windows as the normal way to list directories may report not * updated info. Only the GetFileInformationByHandle() func, called file-by-file, * relly ensures to return synced info. + * + * If this happens, we read also the physical offset, to avoid to read it later. */ -static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st) +static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struct stat* st, uint64_t* physical) { #if HAVE_LSTAT_SYNC + struct snapraid_state* state = scan->state; struct snapraid_disk* disk = scan->disk; /* if the st_sync is not set, ensure to get synced info */ @@ -331,7 +336,16 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc pathprint(path_next, sizeof(path_next), "%s%s", disk->dir, sub); - if (lstat_sync(path_next, &synced_st) != 0) { + /* if we sort for physical offsets we have to read them for new files */ + if (state->opt.force_order == SORT_PHYSICAL + && *physical == FILEPHY_UNREAD_OFFSET + ) { + /* do nothing, leave the pointer to read the physical offset */ + } else { + physical = 0; /* set the pointer to 0 to read nothing */ + } + + if (lstat_sync(path_next, &synced_st, physical) != 0) { /* LCOV_EXCL_START */ fprintf(stderr, "Error in stat file '%s'. %s.\n", path_next, strerror(errno)); exit(EXIT_FAILURE); @@ -809,7 +823,7 @@ static void scan_file(struct snapraid_scan* scan, int output, const char* sub, s /* refresh the info, to ensure that they are synced, */ /* not that we refresh only the info of the new or modified files */ /* because this is slow operation */ - scan_file_refresh(scan, sub, st); + scan_file_refresh(scan, sub, st, &physical); /* do a safety check to ensure that the common ext4 case of zeroing */ /* the size of a file after a crash doesn't propagate to the backup */ @@ -1210,7 +1224,7 @@ static int scan_dir(struct snapraid_scan* scan, int output, const char* dir, con /* if the st_ino field is missing, takes care to fill it using the extended lstat() */ /* this can happen only in Windows */ if (st->st_ino == 0) { - if (lstat_sync(path_next, st) != 0) { + if (lstat_sync(path_next, st, 0) != 0) { /* LCOV_EXCL_START */ fprintf(stderr, "Error in stat file '%s'. %s.\n", path_next, strerror(errno)); exit(EXIT_FAILURE);