rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Uses inodes to identify files.

Browse source 
It allows to automatically handle file move or rename without the need to rehash the file.
This commit is contained in:
Andrea Mazzoleni 2011-03-20 22:36:49 +01:00
parent 8fb43ddefa
commit de07510220
6 changed files with 183 additions and 39 deletions
Download patch file Download diff file Expand all files Collapse all files

17
elem.c
View file

@ -128,7 +128,7 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
return block_size;
}
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, time_t mtime)
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime, uint64_t inode)
{
struct snapraid_file* file;
block_off_t i;
@ -138,6 +138,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
file->size = size;
file->blockmax = (size + block_size - 1) / block_size;
file->mtime = mtime;
file->inode = inode;
file->is_present = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
@ -157,11 +158,15 @@ void file_free(struct snapraid_file* file)
free(file);
}
int file_compare(const void* void_arg, const void* void_data)
int file_inode_compare(const void* void_arg, const void* void_data)
{
const char* arg = void_arg;
const uint64_t* arg = void_arg;
const struct snapraid_file* file = void_data;
return strcmp(arg, file->sub);
if (*arg < file->inode)
return -1;
if (*arg > file->inode)
return 1;
return 0;
}
struct snapraid_disk* disk_alloc(const char* name, const char* dir)
@ -177,7 +182,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir)
disk->first_free_block = 0;
tommy_list_init(&disk->filelist);
tommy_hashdyn_init(&disk->fileset);
tommy_hashdyn_init(&disk->inodeset);
tommy_array_init(&disk->blockarr);
return disk;
@ -191,7 +196,7 @@ void disk_free(struct snapraid_disk* disk)
node = node->next;
file_free(file);
}
tommy_hashdyn_done(&disk->fileset);
tommy_hashdyn_done(&disk->inodeset);
tommy_array_done(&disk->blockarr);
free(disk);
}

17
elem.h
View file

@ -78,7 +78,8 @@ struct snapraid_file {
data_off_t size; /**< Size of the file. */
struct snapraid_block* blockvec; /**< All the blocks of the file. */
block_off_t blockmax; /**< Number of blocks. */
time_t mtime; /**< Modification time. */
uint64_t mtime; /**< Modification time. */
uint64_t inode; /**< Inode. */
int is_present; /**< If it's seen as present. */
/* nodes for data structures */
@ -94,7 +95,7 @@ struct snapraid_disk {
char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
block_off_t first_free_block; /**< First free searching block. */
tommy_list filelist; /**< List of all the files. */
tommy_hashdyn fileset; /**< Hashtable by sub of all the files. */
tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
tommy_array blockarr; /**< Block array of the disk. */
};
@ -128,7 +129,7 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size);
/**
* Allocates a file.
*/
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, time_t mtime);
struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime, uint64_t inode);
/**
* Deallocates a file.
@ -136,16 +137,16 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_
void file_free(struct snapraid_file* file);
/**
* Compares two files by path.
* Compares two files by inode.
*/
int file_compare(const void* void_arg, const void* void_data);
int file_inode_compare(const void* void_arg, const void* void_data);
/**
* Computes the hash of a file path.
* Computes the hash of a file inode.
*/
static inline tommy_uint32_t file_hash(const char* sub)
static inline tommy_uint32_t file_inode_hash(uint64_t inode)
{
return tommy_hash_u32(0, sub, strlen(sub));
return (tommy_uint32_t)tommy_inthash_u64(inode);
}
/**

114
portable.h
View file

@ -113,14 +113,113 @@
#ifdef __MINGW32__ /* Specific for MINGW */
/* Remap functions and types for 64 bit support */
#define stat _stati64
#define stat windows_stat
#define off_t off64_t
#define lseek lseek64
#define fstat _fstati64
#define fstat windows_fstat
#define HAVE_FTRUNCATE 1
#define ftruncate windows_ftruncate
#define HAVE_FSYNC 1
#define fsync _commit
#define rename windows_rename
/**
* Generic stat information.
*/
struct windows_stat {
uint64_t st_ino;
uint64_t st_size;
uint64_t st_mtime;
uint32_t st_mode;
};
/**
* Convert Windows info to the Unix stat format.
*/
static inline void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, struct windows_stat* st)
{
/* Convert special attributes to a char device */
if ((info->dwFileAttributes & (FILE_ATTRIBUTE_DEVICE | FILE_ATTRIBUTE_TEMPORARY | FILE_ATTRIBUTE_OFFLINE | FILE_ATTRIBUTE_VIRTUAL | FILE_ATTRIBUTE_REPARSE_POINT)) != 0) {
st->st_mode = S_IFCHR;
} else if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) {
st->st_mode = S_IFDIR;
} else {
st->st_mode = S_IFREG;
}
st->st_size = info->nFileSizeHigh;
st->st_size <<= 32;
st->st_size |= info->nFileSizeLow;
st->st_mtime = info->ftLastWriteTime.dwHighDateTime;
st->st_mtime <<= 32;
st->st_mtime |= info->ftLastWriteTime.dwLowDateTime;
/*
* Convert to unix time
*
* How To Convert a UNIX time_t to a Win32 FILETIME or SYSTEMTIME
* http://support.microsoft.com/kb/167296
*/
st->st_mtime = (st->st_mtime - 116444736000000000LL) / 10000000;
st->st_ino = info->nFileIndexHigh;
st->st_ino <<= 32;
st->st_ino |= info->nFileIndexLow;
}
static inline int windows_fstat(int fd, struct windows_stat* st)
{
BY_HANDLE_FILE_INFORMATION info;
HANDLE h;
h = (HANDLE)_get_osfhandle(fd);
if (h == INVALID_HANDLE_VALUE) {
errno = EBADF;
return -1;
}
if (!GetFileInformationByHandle(h, &info)) {
errno = EIO;
return -1;
}
windows_info2stat(&info, st);
return 0;
}
static inline int windows_stat(const char* file, struct windows_stat* st)
{
BY_HANDLE_FILE_INFORMATION info;
HANDLE h;
h = CreateFile(file, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, 0, 0);
if (h == INVALID_HANDLE_VALUE) {
DWORD error = GetLastError();
switch (error) {
case ERROR_FILE_NOT_FOUND :
errno = ENOENT;
break;
default:
errno = EIO;
break;
}
return -1;
}
if (!GetFileInformationByHandle(h, &info)) {
CloseHandle(h);
errno = EIO;
return -1;
}
windows_info2stat(&info, st);
CloseHandle(h);
return 0;
}
static inline int windows_ftruncate(int fd, off64_t off)
{
@ -167,12 +266,19 @@ static inline int windows_ftruncate(int fd, off64_t off)
}
return -1;
}
return 0;
}
#define rename windows_atomic_rename
static inline int windows_atomic_rename(const char* a, const char* b)
static inline int windows_rename(const char* a, const char* b)
{
/*
* Implements an atomic rename in Windows.
* Not really atomic at now to support XP.
*
* Is an atomic file rename (with overwrite) possible on Windows?
* http://stackoverflow.com/questions/167414/is-an-atomic-file-rename-with-overwrite-possible-on-windows
*/
if (!MoveFileEx(a, b, MOVEFILE_REPLACE_EXISTING)) {
switch (GetLastError()) {
case ERROR_ACCESS_DENIED :

44
scan.c
View file

@ -47,7 +47,7 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
}
/* remove the file from the file containers */
tommy_hashdyn_remove_existing(&disk->fileset, &file->nodeset);
tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
tommy_list_remove_existing(&disk->filelist, &file->nodelist);
/* deallocate */
@ -87,7 +87,7 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
}
/* insert the file in the file containers */
tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub));
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
}
@ -97,11 +97,19 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* path, const char* sub, const struct stat* st)
{
struct snapraid_file* file;
uint64_t inode;
/* check if the file already exists */
file = tommy_hashdyn_search(&disk->fileset, file_compare, sub, file_hash(sub));
inode = st->st_ino;
file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare, &inode, file_inode_hash(inode));
if (file) {
/* check if the file is the same */
/* check if multiple files have the same inode */
if (file->is_present) {
fprintf(stderr, "Internal inode '%llu' inconsistency for file '%s'\n", inode, sub);
exit(EXIT_FAILURE);
}
/* check if the file is not changed */
if (file->size == st->st_size && file->mtime == st->st_mtime) {
/* mark as present */
++scan->count_equal;
@ -128,7 +136,9 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
/* create the new file */
++scan->count_insert;
file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime);
file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, st->st_ino);
/* mark it as present */
file->is_present = 1;
/* insert it */
@ -168,8 +178,13 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s
while ((dd = readdir(d)) != 0) {
char path_next[PATH_MAX];
char sub_next[PATH_MAX];
const char* name = dd->d_name;
struct stat st;
const char* name = dd->d_name;
int is_auto = name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0));
/* skip "." and ".." files */
if (is_auto)
continue;
pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name);
@ -188,16 +203,13 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s
}
}
} else if (S_ISDIR(st.st_mode)) {
int is_auto = name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0));
if (!is_auto) {
if (scan_filter(state, sub_next, name, 1) == 0) {
pathslash(path_next, sizeof(path_next));
pathslash(sub_next, sizeof(sub_next));
scan_dir(scan, state, disk, path_next, sub_next);
} else {
if (state->verbose) {
printf("warning: Excluded directory '/%s'\n", sub_next);
}
if (scan_filter(state, sub_next, name, 1) == 0) {
pathslash(path_next, sizeof(path_next));
pathslash(sub_next, sizeof(sub_next));
scan_dir(scan, state, disk, path_next, sub_next);
} else {
if (state->verbose) {
printf("warning: Excluded directory '/%s'\n", sub_next);
}
}
} else {

25
state.c
View file

@ -285,9 +285,11 @@ void state_read(struct snapraid_state* state)
char* name;
char* size;
char* mtime;
char* inode;
char* sub;
data_off_t v_size;
uint64_t v_size;
uint64_t v_mtime;
uint64_t v_inode;
unsigned i;
if (file) {
@ -301,6 +303,8 @@ void state_read(struct snapraid_state* state)
s = strtoken(s);
mtime = s;
s = strtoken(s);
inode = s;
s = strtoken(s);
sub = s;
ret = stru64(size, &v_size);
@ -315,6 +319,12 @@ void state_read(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
ret = stru64(inode, &v_inode);
if (ret != 0) {
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
if (!*sub) {
fprintf(stderr, "Invalid 'file' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
@ -332,10 +342,10 @@ void state_read(struct snapraid_state* state)
}
/* allocate the file */
file = file_alloc(state->block_size, sub, v_size, v_mtime);
file = file_alloc(state->block_size, sub, v_size, v_mtime, v_inode);
/* insert the file in the file containers */
tommy_hashdyn_insert(&disk->fileset, &file->nodeset, file, file_hash(file->sub));
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
/* start the block allocation of the file */
@ -418,8 +428,15 @@ void state_write(struct snapraid_state* state)
for(j=disk->filelist;j!=0;j=j->next) {
block_off_t k;
struct snapraid_file* file = j->data;
uint64_t size;
uint64_t mtime;
uint64_t inode;
ret = fprintf(f,"file %s %lld %ld %s\n", disk->name, file->size, file->mtime, file->sub);
size = file->size;
mtime = file->mtime;
inode = file->inode,
ret = fprintf(f,"file %s %llu %llu %llu %s\n", disk->name, size, mtime, inode, file->sub);
if (ret < 0) {
fprintf(stderr, "Error writing the content file '%s' in fprintf(). %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);

5
sync.c
View file

@ -117,7 +117,10 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
}
/* check if the file is changed */
if (handle[j].st.st_size != block->file->size || handle[j].st.st_mtime != block->file->mtime) {
if (handle[j].st.st_size != block->file->size
|| handle[j].st.st_mtime != block->file->mtime
|| handle[j].st.st_ino != block->file->inode
) {
fprintf(stderr, "Unexpected change at file '%s'.\n", handle[j].path);
fprintf(stderr, "WARNING! You cannot modify data disk during a sync. Rerun the sync command when finished.\n");
fprintf(stderr, "Stopping at block %u\n", i);