rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-25 18:40:19 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Restores the modification time of files in fixing.

Browse source
This commit is contained in:
Andrea Mazzoleni 2011-08-31 23:33:43 +02:00
parent e6b02b7d2d
commit 66e9c359fd
15 changed files with 226 additions and 67 deletions
Download patch file Download diff file Expand all files Collapse all files

1
HISTORY
View file

@ -6,6 +6,7 @@ SnapRAID HISTORY
* In "diff" now it recognizes the reuse of inodes.
* Windows hidden files are now backuped like any other files.
* Symbolic links are now saved in *nix. Not supported in Windows.
* When fixing the original modification time of the files is restored.
1.5 2011/7
==========

11
Makefile.am
View file

@ -77,6 +77,8 @@ maintainer-clean-local:
# Automatic testing
CHECKFLAGS =
CHECKCOUNT = 2000
CHECKSIZE = 4000
check-local:
rm -rf test
@ -92,13 +94,13 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check
echo --- Create some files, sync raid6 and check
$(TESTENV) ./mktest$(EXEEXT) generate 1 3 1500 4000
$(TESTENV) ./mktest$(EXEEXT) generate 1 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid5.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check
echo --- Create some more files, sync raid5 and check
$(TESTENV) ./mktest$(EXEEXT) generate 2 3 1500 4000
$(TESTENV) ./mktest$(EXEEXT) generate 2 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid5.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid5.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid5.conf check
@ -114,7 +116,7 @@ check-local:
rm test/disk1/1*
rm test/disk2/2*
rm test/disk3/3*
$(TESTENV) ./mktest$(EXEEXT) generate 3 3 200 4000
$(TESTENV) ./mktest$(EXEEXT) generate 3 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check
echo --- Move some files, sync and check
@ -137,6 +139,7 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -U -c $(srcdir)/test-raid5.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -R -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid5.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check
echo --- Sync after all the fixes with an empty drive to test -E
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -E -c $(srcdir)/test-raid6.conf sync
@ -144,7 +147,7 @@ check-local:
rm test/disk1/4*
rm test/disk2/5*
rm test/disk3/6*
$(TESTENV) ./mktest$(EXEEXT) generate 4 3 200 4000
$(TESTENV) ./mktest$(EXEEXT) generate 4 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf -t 100 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf -t 700 sync

17
TODO
View file

@ -1,7 +1,7 @@
SnapRAID TODO
=============
This is a list of possible TODO items for SnapRAID. The order is not relevant.
This is a list of possible TODO items for SnapRAID.
* When fixing, before overwriting the present file, make a copy of it just in
case that the original file cannot be completely recovered.
@ -14,25 +14,20 @@ the file name cannot be supported, it's better to do not stop the sync process.
The sync process should stop only if the error could be caused by a disk
corruption.
* In the speed test prints also the whole memory available in the system.
I may help to detect the slowdown cases.
* Adds a checksum for the content file. When reading if the file is invalid,
skip to another.
* Easy way to find duplicates using the computed hash.
* Easy way to find files using a specified block.
* Add an option to exclude "hidden" file. In Windows files with the HIDDEN attributes,
in Unix files starting with '.'.
* Adds a checksum for the content file. When reading if the file is invalid,
skip to another.
* Implement an "undelete" command to recover missing files, without the
integrity check of the existing ones.
It should restore only missing files, without recovering modified ones
or check validity of data of existing ones.
* When fixing also restore the file mtime. The timestamp is also important.
* Add an option to exclude "hidden" file. In Windows files with the HIDDEN attributes,
in Unix files starting with '.'.
* Allow to filter the files to check/fix based on the drive where they reside.
@ -45,7 +40,7 @@ better the meaning of the commands. But not yet sure.
If they do not match, stop the processing.
This can be done to avoid to use not syncronized parity and content files,
resulting in wrong data.
But take care that if the sync process is killed we need a way to resyncronize them.
But if the sync process is killed we need a way to resyncronize them.
Rejected TODO

101
check.c
View file

@ -85,14 +85,14 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
/* recover */
raid5_recov_data(buffer, diskmax, state->block_size, failed[0].index);
/* check if the recovered blocks are OK */
for(j=0;j<1;++j) {
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
}
if (j==1) {
/* compute parity and qarity to check it */
/* we recompute everything because we may have used only a small part of the redundancy */
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
@ -107,14 +107,14 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
raid6_recov_datap(buffer, diskmax, state->block_size, failed[0].index, buffer_zero);
/* check if the recovered blocks are OK */
for(j=0;j<1;++j) {
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
}
if (j==1) {
/* compute parity and qarity to check it */
/* we recompute everything because we may have used only a small part of the redundancy */
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
@ -131,14 +131,14 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
/* recover */
raid6_recov_2data(buffer, diskmax, state->block_size, failed[0].index, failed[1].index, buffer_zero);
/* check if the recovered blocks are OK */
for(j=0;j<2;++j) {
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
}
if (j==2) {
/* compute parity and qarity to check it */
/* we recompute everything because we may have used only a small part of the redundancy */
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
@ -281,12 +281,15 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
if (!block_flag_has(block, BLOCK_HAS_PARITY))
all_parity = 0;
ret = handle_close_if_different(&handle[j], block_file_get(block));
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n");
printf("Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {
ret = handle_close(&handle[j]);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n");
printf("Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
}
if (fix) {
@ -417,9 +420,15 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
/* increment the number of errors */
if (ret > 0)
error += ret;
++unrecoverable_error;
if (fix) {
/* keep track of damaged files */
for(j=0;j<failed_count;++j) {
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
}
}
/* print a list of all the errors in files */
for(j=0;j<failed_count;++j) {
fprintf(stderr, "error:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
@ -456,6 +465,9 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size);
if (ret == -1) {
/* mark the file as damaged */
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
if (errno == EACCES) {
fprintf(stderr, "WARNING! Please give write permission to the file.\n");
} else {
@ -467,6 +479,9 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
goto bail;
}
/* mark the file as fixed */
file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED);
fprintf(stderr, "fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
++recovered_error;
}
@ -510,6 +525,66 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
}
if (fix) {
/* for all the files of this block check if we need to fix the modification time */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block;
struct snapraid_file* collide_file;
uint64_t inode;
block = disk_block_get(handle[j].disk, i);
if (!block) {
/* if no block, no file and nothing to do */
continue;
}
/* if it isn't the last block in the file */
if (!block_is_last(block)) {
/* nothing to do */
continue;
}
/* if the file is not fixed or if it's not fixable */
if (!file_flag_has(handle[j].file, FILE_IS_FIXED)
|| file_flag_has(handle[j].file, FILE_IS_DAMAGED)) {
/* nothing to do */
continue;
}
inode = handle[j].st.st_ino;
/* search for the corresponding inode */
collide_file = tommy_hashdyn_search(&handle[j].disk->inodeset, file_inode_compare, &inode, file_inode_hash(inode));
/* if the inode is in the database and it refers at a different file name, */
/* we can fix the file time ONLY if the time and size allow to differentiates */
/* between the two files */
/* for example, suppose we delete a bunch of files with all the same size and time, */
/* when recreating them the inodes may be reused in a different order, */
/* and at the next sync some files may have matching inode/size/time even if different name */
/* not allowing sync to detect that the file is changed and not renamed */
if (!collide_file /* if not in the databse, there is no collision */
|| strcmp(collide_file->sub, handle[j].file->sub) == 0 /* if the name is the same, it's the right collision */
|| collide_file->size != handle[j].file->size /* if the size is different, the collision is identified */
|| collide_file->mtime != handle[j].file->mtime /* if the mtime is different, the collision is identified */
) {
/* set the original modification time */
ret = handle_utime(&handle[j]);
if (ret == -1) {
/* mark the file as damaged */
file_flag_set(handle[j].file, FILE_IS_DAMAGED);
fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
printf("Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
} else {
fprintf(stderr, "collision:%s:%s:%s: Not setting modification time to avoid inode collision\n", handle[j].disk->name, block_file_get(block)->sub, collide_file->sub);
}
}
}
/* count the number of processed block */
++countpos;

13
dry.c
View file

@ -63,11 +63,14 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, int qar
if (!block)
continue;
ret = handle_close_if_different(&handle[j], block_file_get(block));
if (ret == -1) {
printf("Stopping at block %u\n", i);
++error;
goto bail;
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {
ret = handle_close(&handle[j]);
if (ret == -1) {
printf("Stopping at block %u\n", i);
++error;
goto bail;
}
}
/* open the file for reading */

7
elem.c
View file

@ -193,6 +193,13 @@ block_off_t block_file_pos(struct snapraid_block* block)
return block - file->blockvec;
}
int block_is_last(struct snapraid_block* block)
{
struct snapraid_file* file = block_file_get(block);
return block == file->blockvec + file->blockmax - 1;
}
unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
{
block_off_t pos = block_file_pos(block);

18
elem.h
View file

@ -67,6 +67,7 @@ struct snapraid_file;
#define BLOCK_HAS_HASH 1 /**< If the hash value is valid. */
#define BLOCK_HAS_PARITY 2 /**< If the parity block is valid. */
#define BLOCK_HAS_MASK 3 /**< Mask of all the flags. Used to mix them inside the file pointer. */
/**
* Block of a file.
@ -80,6 +81,8 @@ struct snapraid_block {
#define FILE_IS_PRESENT 1 /**< If it's seen as present. */
#define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */
#define FILE_IS_LARGER 4 /**< If a larger file was already detected. */
#define FILE_IS_DAMAGED 8 /**< If a fix was attempted but it failed. */
#define FILE_IS_FIXED 16 /**< If a fix was done. */
/**
* File.
@ -152,14 +155,20 @@ void filter_free(struct snapraid_filter* filter);
*/
int filter_path(tommy_list* filterlist, const char* path, int is_dir);
/**
* Gets the file containing the block.
*/
static inline struct snapraid_file* block_file_get(struct snapraid_block* block)
{
return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)3);
return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)BLOCK_HAS_MASK);
}
/**
* Sets the file containing the block.
*/
static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file)
{
block->file_mixed = (block->file_mixed & ~(uintptr_t)3) | (uintptr_t)file;
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_HAS_MASK) | (uintptr_t)file;
}
static inline int block_flag_has(struct snapraid_block* block, unsigned mask)
@ -182,6 +191,11 @@ static inline void block_flag_clear(struct snapraid_block* block, unsigned mask)
*/
block_off_t block_file_pos(struct snapraid_block* block);
/**
* Checks if the block is the last in the file.
*/
int block_is_last(struct snapraid_block* block);
/**
* Gets the size in bytes of the block.
* If it's the last block of a file it could be less than block_size.

33
handle.c
View file

@ -226,16 +226,6 @@ int handle_close(struct snapraid_handle* handle)
return 0;
}
int handle_close_if_different(struct snapraid_handle* handle, struct snapraid_file* file)
{
/* if it's the same file, nothing to do */
if (handle->file == file) {
return 0;
}
return handle_close(handle);
}
int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size)
{
ssize_t read_ret;
@ -309,4 +299,27 @@ int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, u
return 0;
}
int handle_utime(struct snapraid_handle* handle)
{
struct timeval tv[2];
int ret;
/* do nothing if not opened */
if (handle->f == -1)
return 0;
tv[0].tv_sec = handle->file->mtime;
tv[0].tv_usec = 0;
tv[1].tv_sec = tv[0].tv_sec;
tv[1].tv_usec = tv[0].tv_usec;
ret = futimes(handle->f, tv);
if (ret != 0) {
fprintf(stderr, "Error timing file '%s'. %s.\n", handle->file->sub, strerror(errno));
return -1;
}
return 0;
}

10
handle.h
View file

@ -34,11 +34,6 @@ struct snapraid_handle {
*/
int handle_ancestor(const char* file);
/**
* Closes a file, if it's different from the specified one.
*/
int handle_close_if_different(struct snapraid_handle* handle, struct snapraid_file* file);
/**
* Creates a file.
* The file is created if missing, and opened with write access.
@ -72,5 +67,10 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un
*/
int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size);
/**
* Changes the modification time of the file to the saved value.
*/
int handle_utime(struct snapraid_handle* handle);
#endif

39
mingw.c
View file

@ -318,6 +318,45 @@ int windows_ftruncate(int fd, off64_t off)
return 0;
}
int windows_futimes(int fd, struct timeval tv[2])
{
HANDLE h;
FILETIME ft;
uint64_t mtime;
if (fd == -1) {
errno = EBADF;
return -1;
}
h = (HANDLE)_get_osfhandle(fd);
if (h == INVALID_HANDLE_VALUE) {
errno = EBADF;
return -1;
}
/*
* Convert to windows time
*
* How To Convert a UNIX time_t to a Win32 FILETIME or SYSTEMTIME
* http://support.microsoft.com/kb/167296
*/
mtime = tv[0].tv_sec;
mtime *= 10000000;
mtime += tv[0].tv_usec * 10;
mtime += 116444736000000000;
ft.dwHighDateTime = mtime >> 32;
ft.dwLowDateTime = mtime;
if (!SetFileTime(h, 0, 0, &ft)) {
windows_errno(GetLastError());
return -1;
}
return 0;
}
int windows_rename(const char* a, const char* b)
{
/*

12
mingw.h
View file

@ -49,6 +49,7 @@
#define opendir windows_opendir
#define readdir windows_readdir
#define closedir windows_closedir
#define futimes windows_futimes
#define O_NOFOLLOW 0
#define S_ISLNK(mode) 0
#define readlink(a,b,c) -1
@ -66,12 +67,12 @@ struct windows_stat {
};
/**
* Like the C fstat() but without inode information.
* Like the C fstat() including the inode information.
*/
int windows_fstat(int fd, struct windows_stat* st);
/**
* Like the C stat() but without inode information.
* Like the C stat() but without the inode information.
*/
int windows_lstat(const char* file, struct windows_stat* st);
@ -86,7 +87,7 @@ int windows_access(const char* file, int mode);
int windows_mkdir(const char* file);
/**
* Like the C lstat() with inode information.
* Like the C lstat() including the inode information.
* It doesn't work for all kind of files and directories. For example "\System Volume Information" cannot be opened.
* Note that instead lstat() works for all the files.
*/
@ -98,6 +99,11 @@ int lstat_inode(const char* file, struct windows_stat* st);
*/
int windows_ftruncate(int fd, off64_t off);
/**
* Like the C futimes().
*/
int windows_futimes(int fd, struct timeval tv[2]);
/**
* Like the C rename().
*/

4
snapraid.1
View file

@ -101,8 +101,8 @@ It\(cqs able to recover damages only from up to two disks.
With a Backup you are able to recover from a complete
failure of the whole disk array.
.IP \(bu
Only file data and symlinks are saved. Permissions, time, extended attributes,
hard\(hylinks are not saved.
Only file data, time and symlinks are saved. Permissions,
extended attributes and hard\(hylinks are not saved.
.PD
.SH GETTING STARTED
To use SnapRAID you need first select one disk of your disk array

4
snapraid.d
View file

@ -67,8 +67,8 @@ Limitations
* It's able to recover damages only from up to two disks.
With a Backup you are able to recover from a complete
failure of the whole disk array.
* Only file data and symlinks are saved. Permissions, time, extended attributes,
hard-links are not saved.
* Only file data, time and symlinks are saved. Permissions,
extended attributes and hard-links are not saved.
Getting Started
To use SnapRAID you need first select one disk of your disk array

4
snapraid.txt
View file

@ -77,8 +77,8 @@ Other limitations are:
* It's able to recover damages only from up to two disks.
With a Backup you are able to recover from a complete
failure of the whole disk array.
* Only file data and symlinks are saved. Permissions, time, extended attributes,
hard-links are not saved.
* Only file data, time and symlinks are saved. Permissions,
extended attributes and hard-links are not saved.
4 GETTING STARTED

19
sync.c
View file

@ -116,14 +116,17 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
continue;
}
ret = handle_close_if_different(&handle[j], block_file_get(block));
if (ret == -1) {
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n");
printf("Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {
ret = handle_close(&handle[j]);
if (ret == -1) {
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n");
printf("Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
}
ret = handle_open(&handle[j], block_file_get(block));