Sorts the files by physical address before inserting them for the first time.

This commit is contained in:
Andrea Mazzoleni 2013-06-25 22:41:10 +02:00
parent e0164b29d1
commit 6032576650
12 changed files with 245 additions and 18 deletions

View file

@ -15,6 +15,8 @@ SnapRAID HISTORY
* Automatically ignores, with warning, mount points inside the array
directory tree.
* Changes the 'dup' output format to include the size of each duplicate file.
* When insering files for the first time, sort them by their physical
address to improve read performance.
3.1 2013/5
==========

View file

@ -1017,7 +1017,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
inode = handle[j].st.st_ino;
/* search for the corresponding inode */
collide_file = tommy_hashdyn_search(&handle[j].disk->inodeset, file_inode_compare, &inode, file_inode_hash(inode));
collide_file = tommy_hashdyn_search(&handle[j].disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
/* if the inode is already in the database and it refers at a different file name, */
/* we can fix the file time ONLY if the time and size allow to differentiates */

View file

@ -23,7 +23,8 @@ AC_HEADER_STDC
AC_HEADER_DIRENT
AC_HEADER_TIME
AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h limits.h])
AC_CHECK_HEADERS([sys/time.h sys/file.h unistd.h getopt.h fnmatch.h io.h inttypes.h byteswap.h])
AC_CHECK_HEADERS([unistd.h getopt.h fnmatch.h io.h inttypes.h byteswap.h])
AC_CHECK_HEADERS([sys/time.h sys/file.h sys/ioctl.h linux/fiemap.h linux/fs.h])
dnl Checks for typedefs, structures, and compiler characteristics.
AC_C_CONST

34
elem.c
View file

@ -339,7 +339,7 @@ void deleted_free(struct snapraid_deleted* deleted)
free(deleted);
}
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode)
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical)
{
struct snapraid_file* file;
block_off_t i;
@ -351,6 +351,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t
file->mtime_sec = mtime_sec;
file->mtime_nsec = mtime_nsec;
file->inode = inode;
file->physical = physical;
file->flag = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
@ -387,7 +388,7 @@ const char* file_name(struct snapraid_file* file)
return r;
}
int file_inode_compare(const void* void_arg, const void* void_data)
int file_inode_compare_to_arg(const void* void_arg, const void* void_data)
{
const uint64_t* arg = void_arg;
const struct snapraid_file* file = void_data;
@ -398,6 +399,35 @@ int file_inode_compare(const void* void_arg, const void* void_data)
return 0;
}
int file_inode_compare(const void* void_a, const void* void_b)
{
const struct snapraid_file* file_a = void_a;
const struct snapraid_file* file_b = void_b;
if (file_a->inode < file_b->inode)
return -1;
if (file_a->inode > file_b->inode)
return 1;
return 0;
}
int file_alpha_compare(const void* void_a, const void* void_b)
{
const struct snapraid_file* file_a = void_a;
const struct snapraid_file* file_b = void_b;
return strcmp(file_a->sub, file_b->sub);
}
int file_physical_compare(const void* void_a, const void* void_b)
{
const struct snapraid_file* file_a = void_a;
const struct snapraid_file* file_b = void_b;
if (file_a->physical < file_b->physical)
return -1;
if (file_a->physical > file_b->physical)
return 1;
return 0;
}
int file_path_compare(const void* void_arg, const void* void_data)
{
const char* arg = void_arg;

20
elem.h
View file

@ -201,6 +201,7 @@ struct snapraid_block {
struct snapraid_file {
int64_t mtime_sec; /**< Modification time. */
uint64_t inode; /**< Inode. */
uint64_t physical; /**< Physical offset of the file. */
data_off_t size; /**< Size of the file. */
struct snapraid_block* blockvec; /**< All the blocks of the file. */
int mtime_nsec; /**< Modification time nanoseconds. In the range 0 <= x < 1,000,000,000, or FILE_MTIME_NSEC_INVALID. */
@ -516,7 +517,7 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
/**
* Allocates a file.
*/
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode);
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical);
/**
* Deallocates a file.
@ -536,7 +537,22 @@ const char* file_name(struct snapraid_file* file);
/**
* Compares a file with an inode.
*/
int file_inode_compare(const void* void_arg, const void* void_data);
int file_inode_compare_to_arg(const void* void_arg, const void* void_data);
/**
* Compares files by inode.
*/
int file_inode_compare(const void* void_a, const void* void_b);
/**
* Compares files by path.
*/
int file_alpha_compare(const void* void_a, const void* void_b);
/**
* Compares files by physical address.
*/
int file_physical_compare(const void* void_a, const void* void_b);
/**
* Computes the hash of a file inode.

58
mingw.c
View file

@ -48,6 +48,21 @@ typedef struct _FILE_ATTRIBUTE_TAG_INFO {
#define WIN32_ES_AWAYMODE_REQUIRED 0x00000040L
#define WIN32_ES_CONTINUOUS 0x80000000L
#define FSCTL_GET_RETRIEVAL_POINTERS 0x00090073
typedef struct RETRIEVAL_POINTERS_BUFFER {
DWORD ExtentCount;
LARGE_INTEGER StartingVcn;
struct {
LARGE_INTEGER NextVcn;
LARGE_INTEGER Lcn;
} Extents[1];
} RETRIEVAL_POINTERS_BUFFER;
typedef struct {
LARGE_INTEGER StartingVcn;
} STARTING_VCN_INPUT_BUFFER;
/**
* Portable implementation of GetFileInformationByHandleEx.
* This function is not available in Windows 2000.
@ -1062,6 +1077,49 @@ int devuuid(uint64_t device, char* uuid, size_t uuid_size)
return 0;
}
int filephy(const char* file, struct stat* st, uint64_t* physical)
{
HANDLE h;
STARTING_VCN_INPUT_BUFFER svib;
unsigned char rpb_buffer[sizeof(RETRIEVAL_POINTERS_BUFFER)];
RETRIEVAL_POINTERS_BUFFER* rpb = (RETRIEVAL_POINTERS_BUFFER*)&rpb_buffer;
DWORD ret;
DWORD n;
/* open the handle of the file */
h = CreateFileW(convert(file), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, 0, 0);
if (h == INVALID_HANDLE_VALUE) {
windows_errno(GetLastError());
return -1;
}
/* set the output variable, just to be safe */
rpb->ExtentCount = 0;
/* read the physical address */
svib.StartingVcn.QuadPart = 0;
ret = DeviceIoControl(h, FSCTL_GET_RETRIEVAL_POINTERS, &svib, sizeof(svib), rpb_buffer, sizeof(rpb_buffer), &n, 0);
if (ret == 0) {
/* we ignore ERROR_MODE_DATA because we are interested only at the first entry */
if (GetLastError() != ERROR_MORE_DATA) {
windows_errno(GetLastError());
CloseHandle(h);
return -1;
}
}
CloseHandle(h);
if (rpb->ExtentCount < 1)
*physical = 0;
else
*physical = rpb->Extents[0].Lcn.QuadPart;
(void)st; /* not used here */
return 0;
}
#undef strerror
const char* windows_strerror(int err)

View file

@ -70,6 +70,18 @@
#include <sys/file.h>
#endif
#if HAVE_SYS_IOCTL_H
#include <sys/ioctl.h>
#endif
#if HAVE_LINUX_FS_H
#include <linux/fs.h>
#endif
#if HAVE_LINUX_FIEMAP_H
#include <linux/fiemap.h>
#endif
#if HAVE_DIRENT_H
#include <dirent.h>
#define NAMLEN(dirent) strlen((dirent)->d_name)
@ -154,6 +166,13 @@ static inline int hardlink(const char* a, const char* b)
*/
int devuuid(uint64_t device, char* uuid, size_t size);
/**
* Get the physcal address of the specified file.
* This is expected to be just a hint and not necessarely correct or unique.
* Returns 0 on success.
*/
int filephy(const char* path, struct stat* st, uint64_t* physical);
/**
* Initializes the system.
*/

43
scan.c
View file

@ -270,7 +270,7 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
/**
* Processes a file.
*/
static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* sub, const struct stat* st)
static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* sub, const struct stat* st, uint64_t physical)
{
struct snapraid_file* file;
@ -280,7 +280,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
} else {
/* check if the file inode already exists */
uint64_t inode = st->st_ino;
file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare, &inode, file_inode_hash(inode));
file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
}
if (file) {
@ -346,14 +346,15 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
state->need_write = 1;
} else if (file->inode != st->st_ino) {
/* if the inode is different, it means a rewritten file with the same path */
/* like when restoring a backup that restores also the time information */
++scan->count_moved;
if (state->gui) {
fprintf(stdlog, "scan:move:%s:%s:%s\n", disk->name, file->sub, sub);
fprintf(stdlog, "scan:restore:%s:%s\n", disk->name, sub);
fflush(stdlog);
}
if (output) {
printf("Move '%s%s' '%s%s'\n", disk->dir, file->sub, disk->dir, sub);
printf("Restore '%s%s'\n", disk->dir, sub);
}
/* remove from the set */
@ -365,7 +366,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
/* reinsert in the set */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
/* we have to save the new name */
/* we have to save the new inode */
state->need_write = 1;
} else {
/* otherwise it's equal */
@ -446,7 +447,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
}
/* insert it */
file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino);
file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, STAT_NSEC(st), st->st_ino, physical);
/* mark it as present */
file_flag_set(file, FILE_IS_PRESENT);
@ -591,7 +592,7 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
exit(EXIT_FAILURE);
}
/* exclude hidden files even beforer calling lstat() */
/* exclude hidden files even before calling lstat() */
if (filter_hidden(state->filter_hidden, dd) != 0) {
if (state->verbose) {
printf("Excluding hidden '%s'\n", path_next);
@ -620,6 +621,8 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
if (S_ISREG(st.st_mode)) {
if (filter_path(&state->filterlist, disk->name, sub_next) == 0) {
uint64_t physical;
#if HAVE_LSTAT_EX
/* get inode info about the file, Windows needs an additional step */
/* also for hardlink, the real size of the file is read here */
@ -628,8 +631,16 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in
exit(EXIT_FAILURE);
}
#endif
if (state->force_order == SORT_PHYSICAL) {
if (filephy(path_next, &st, &physical) != 0) {
fprintf(stderr, "Error in getting the physical offset of file '%s'. %s.\n", path_next, strerror(errno));
exit(EXIT_FAILURE);
}
} else {
physical = 0;
}
scan_file(scan, state, output, disk, sub_next, &st);
scan_file(scan, state, output, disk, sub_next, &st, physical);
processed = 1;
} else {
if (state->verbose) {
@ -805,6 +816,22 @@ void state_scan(struct snapraid_state* state, int output)
}
}
/* sort the files before inserting them */
switch (state->force_order) {
case SORT_PHYSICAL :
tommy_list_sort(&scan->file_insert_list, file_physical_compare);
break;
case SORT_INODE :
tommy_list_sort(&scan->file_insert_list, file_inode_compare);
break;
case SORT_ALPHA :
tommy_list_sort(&scan->file_insert_list, file_alpha_compare);
break;
case SORT_DIR :
/* already in order */
break;
}
/* insert all the new files, we insert them only after the deletion */
/* to reuse the just freed space */
node = scan->file_insert_list;

View file

@ -85,6 +85,10 @@ void usage(void)
#define OPT_TEST_FORCE_MURMUR3 264
#define OPT_TEST_FORCE_SPOOKY2 265
#define OPT_TEST_SKIP_LOCK 266
#define OPT_TEST_FORCE_ORDER_PHYSICAL 267
#define OPT_TEST_FORCE_ORDER_INODE 268
#define OPT_TEST_FORCE_ORDER_ALPHA 269
#define OPT_TEST_FORCE_ORDER_DIR 270
#if HAVE_GETOPT_LONG
struct option long_options[] = {
@ -142,6 +146,12 @@ struct option long_options[] = {
/* Skip the use of lock file */
{ "test-skip-lock", 0, 0, OPT_TEST_SKIP_LOCK },
/* Force a sort order for files */
{ "test-force-order-physical", 0, 0, OPT_TEST_FORCE_ORDER_PHYSICAL },
{ "test-force-order-inode", 0, 0, OPT_TEST_FORCE_ORDER_INODE },
{ "test-force-order-alpha", 0, 0, OPT_TEST_FORCE_ORDER_ALPHA },
{ "test-force-order-dir", 0, 0, OPT_TEST_FORCE_ORDER_DIR },
{ 0, 0, 0, 0 }
};
#endif
@ -189,6 +199,7 @@ int main(int argc, char* argv[])
int test_force_murmur3;
int test_force_spooky2;
int test_skip_lock;
int test_force_order;
const char* conf;
struct snapraid_state state;
int operation;
@ -226,6 +237,7 @@ int main(int argc, char* argv[])
test_force_murmur3 = 0;
test_force_spooky2 = 0;
test_skip_lock = 0;
test_force_order = SORT_PHYSICAL;
blockstart = 0;
blockcount = 0;
tommy_list_init(&filterlist_file);
@ -357,6 +369,18 @@ int main(int argc, char* argv[])
case OPT_TEST_SKIP_LOCK :
test_skip_lock = 1;
break;
case OPT_TEST_FORCE_ORDER_PHYSICAL :
test_force_order = SORT_PHYSICAL;
break;
case OPT_TEST_FORCE_ORDER_INODE :
test_force_order = SORT_INODE;
break;
case OPT_TEST_FORCE_ORDER_ALPHA :
test_force_order = SORT_ALPHA;
break;
case OPT_TEST_FORCE_ORDER_DIR :
test_force_order = SORT_DIR;
break;
default:
fprintf(stderr, "Unknown option '%c'\n", (char)c);
exit(EXIT_FAILURE);
@ -443,7 +467,7 @@ int main(int argc, char* argv[])
state_init(&state);
/* read the configuration file */
state_config(&state, conf, command, verbose, gui, force_zero, force_empty, force_uuid, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_sign, test_skip_fallocate, test_skip_sequential, test_skip_device, test_force_murmur3, test_force_spooky2);
state_config(&state, conf, command, verbose, gui, force_zero, force_empty, force_uuid, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_sign, test_skip_fallocate, test_skip_sequential, test_skip_device, test_force_murmur3, test_force_spooky2, test_force_order);
#if HAVE_LOCKFILE
/* create the lock file */

View file

@ -245,7 +245,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, i
}
}
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2)
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2, int force_order)
{
STREAM* f;
unsigned line;
@ -266,6 +266,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
state->command = command;
state->force_murmur3 = force_murmur3;
state->force_spooky2 = force_spooky2;
state->force_order = force_order;
if (state->gui) {
fprintf(stdlog, "version:%s\n", PACKAGE_VERSION);
@ -1202,7 +1203,7 @@ void state_read(struct snapraid_state* state)
}
/* allocate the file */
file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode);
file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
/* insert the file in the file containers */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));

View file

@ -26,6 +26,11 @@
*/
extern volatile int global_interrupt;
#define SORT_PHYSICAL 1 /**< Sort by physical order. */
#define SORT_INODE 2 /**< Sort by inode. */
#define SORT_ALPHA 3 /**< Sort by alphabetic order. */
#define SORT_DIR 4 /**< Sort by directory order. */
struct snapraid_state {
int verbose; /**< Verbose output. */
int gui; /**< Gui output. */
@ -42,6 +47,7 @@ struct snapraid_state {
int skip_sequential; /**< Skip sequential hint. */
int force_murmur3; /**< Force Murmur3 choice. */
int force_spooky2; /**< Force Spooky2 choice. */
int force_order; /**< Force sorting order. */
int need_write; /**< If the state is changed. */
uint32_t block_size; /**< Block size in bytes. */
char parity[PATH_MAX]; /**< Path of the parity file. */
@ -83,7 +89,7 @@ void state_done(struct snapraid_state* state);
/**
* Reads the configuration file.
*/
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2);
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2, int force_order);
/**
* Reads the state.

43
unix.c
View file

@ -123,6 +123,49 @@ int devuuid(uint64_t device, char* uuid, size_t uuid_size)
}
#endif
int filephy(const char* path, struct stat* st, uint64_t* physical)
{
#if HAVE_LINUX_FIEMAP_H
/* In Linux gets the real physical address of the file */
/* Note that FIEMAP doesn't require root permission */
int f;
struct {
struct fiemap fiemap;
struct fiemap_extent extent;
} fm;
f = open(path, O_RDONLY);
if (f == -1) {
return -1;
}
memset(&fm, 0, sizeof(fm));
fm.fiemap.fm_start = 0;
fm.fiemap.fm_length = -1;
fm.fiemap.fm_flags = FIEMAP_FLAG_SYNC; /* required to ensure that just created files report a valid address and not 0 */
fm.fiemap.fm_extent_count = 1; /* we are interested only at the first block */
if (ioctl(f, FS_IOC_FIEMAP, &fm) == -1) {
close(f);
return -1;
}
*physical = fm.fiemap.fm_extents[0].fe_physical;
if (close(f) == -1) {
return -1;
}
(void)st; /* not used here */
#else
/* In a generic Unix uses the inode as fake physical address */
physical = st->st_ino;
#endif
return 0;
}
void os_init(void)
{
}