diff --git a/HISTORY b/HISTORY index aecffe24..b16b53c1 100644 --- a/HISTORY +++ b/HISTORY @@ -6,6 +6,7 @@ SnapRAID HISTORY * In "diff" now it recognizes the reuse of inodes. * Windows hidden files are now backuped like any other files. * Symbolic links are now saved in *nix. Not supported in Windows. +* When fixing the original modification time of the files is restored. 1.5 2011/7 ========== diff --git a/Makefile.am b/Makefile.am index 2ab73b14..2f00d5ec 100644 --- a/Makefile.am +++ b/Makefile.am @@ -77,6 +77,8 @@ maintainer-clean-local: # Automatic testing CHECKFLAGS = +CHECKCOUNT = 2000 +CHECKSIZE = 4000 check-local: rm -rf test @@ -92,13 +94,13 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check echo --- Create some files, sync raid6 and check - $(TESTENV) ./mktest$(EXEEXT) generate 1 3 1500 4000 + $(TESTENV) ./mktest$(EXEEXT) generate 1 3 $(CHECKCOUNT) $(CHECKSIZE) $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid5.conf check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check echo --- Create some more files, sync raid5 and check - $(TESTENV) ./mktest$(EXEEXT) generate 2 3 1500 4000 + $(TESTENV) ./mktest$(EXEEXT) generate 2 3 $(CHECKCOUNT) $(CHECKSIZE) $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid5.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid5.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid5.conf check @@ -114,7 +116,7 @@ check-local: rm test/disk1/1* rm test/disk2/2* rm test/disk3/3* - $(TESTENV) ./mktest$(EXEEXT) generate 3 3 200 4000 + $(TESTENV) ./mktest$(EXEEXT) generate 3 3 $(CHECKCOUNT) $(CHECKSIZE) $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check echo --- Move some files, sync and check @@ -137,6 +139,7 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -U -c $(srcdir)/test-raid5.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -R -c $(srcdir)/test-raid6.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf fix 2>> test.log + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid5.conf check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check echo --- Sync after all the fixes with an empty drive to test -E $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -E -c $(srcdir)/test-raid6.conf sync @@ -144,7 +147,7 @@ check-local: rm test/disk1/4* rm test/disk2/5* rm test/disk3/6* - $(TESTENV) ./mktest$(EXEEXT) generate 4 3 200 4000 + $(TESTENV) ./mktest$(EXEEXT) generate 4 3 $(CHECKCOUNT) $(CHECKSIZE) $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf -t 100 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -c $(srcdir)/test-raid6.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -S -v -c $(srcdir)/test-raid6.conf -t 700 sync diff --git a/TODO b/TODO index ffbdd983..d0d7a19f 100644 --- a/TODO +++ b/TODO @@ -1,7 +1,7 @@ SnapRAID TODO ============= -This is a list of possible TODO items for SnapRAID. The order is not relevant. +This is a list of possible TODO items for SnapRAID. * When fixing, before overwriting the present file, make a copy of it just in case that the original file cannot be completely recovered. @@ -14,25 +14,20 @@ the file name cannot be supported, it's better to do not stop the sync process. The sync process should stop only if the error could be caused by a disk corruption. -* In the speed test prints also the whole memory available in the system. -I may help to detect the slowdown cases. - -* Adds a checksum for the content file. When reading if the file is invalid, -skip to another. - * Easy way to find duplicates using the computed hash. * Easy way to find files using a specified block. -* Add an option to exclude "hidden" file. In Windows files with the HIDDEN attributes, -in Unix files starting with '.'. +* Adds a checksum for the content file. When reading if the file is invalid, +skip to another. * Implement an "undelete" command to recover missing files, without the integrity check of the existing ones. It should restore only missing files, without recovering modified ones or check validity of data of existing ones. -* When fixing also restore the file mtime. The timestamp is also important. +* Add an option to exclude "hidden" file. In Windows files with the HIDDEN attributes, +in Unix files starting with '.'. * Allow to filter the files to check/fix based on the drive where they reside. @@ -45,7 +40,7 @@ better the meaning of the commands. But not yet sure. If they do not match, stop the processing. This can be done to avoid to use not syncronized parity and content files, resulting in wrong data. -But take care that if the sync process is killed we need a way to resyncronize them. +But if the sync process is killed we need a way to resyncronize them. Rejected TODO diff --git a/check.c b/check.c index 51764169..4f4be9bb 100644 --- a/check.c +++ b/check.c @@ -85,14 +85,14 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st /* recover */ raid5_recov_data(buffer, diskmax, state->block_size, failed[0].index); + /* check if the recovered blocks are OK */ for(j=0;j<1;++j) { if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0) break; } if (j==1) { - /* compute parity and qarity to check it */ - /* we recompute everything because we may have used only a small part of the redundancy */ + /* recompute all the redundancy information */ raid_gen(state->level, buffer, diskmax, state->block_size); return 0; } @@ -107,14 +107,14 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st raid6_recov_datap(buffer, diskmax, state->block_size, failed[0].index, buffer_zero); + /* check if the recovered blocks are OK */ for(j=0;j<1;++j) { if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0) break; } if (j==1) { - /* compute parity and qarity to check it */ - /* we recompute everything because we may have used only a small part of the redundancy */ + /* recompute all the redundancy information */ raid_gen(state->level, buffer, diskmax, state->block_size); return 0; } @@ -131,14 +131,14 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st /* recover */ raid6_recov_2data(buffer, diskmax, state->block_size, failed[0].index, failed[1].index, buffer_zero); + /* check if the recovered blocks are OK */ for(j=0;j<2;++j) { if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0) break; } if (j==2) { - /* compute parity and qarity to check it */ - /* we recompute everything because we may have used only a small part of the redundancy */ + /* recompute all the redundancy information */ raid_gen(state->level, buffer, diskmax, state->block_size); return 0; } @@ -281,12 +281,15 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity if (!block_flag_has(block, BLOCK_HAS_PARITY)) all_parity = 0; - ret = handle_close_if_different(&handle[j], block_file_get(block)); - if (ret == -1) { - fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n"); - printf("Stopping at block %u\n", i); - ++unrecoverable_error; - goto bail; + /* if the file is different than the current one, close it */ + if (handle[j].file != block_file_get(block)) { + ret = handle_close(&handle[j]); + if (ret == -1) { + fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n"); + printf("Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } } if (fix) { @@ -417,9 +420,15 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity /* increment the number of errors */ if (ret > 0) error += ret; - ++unrecoverable_error; + if (fix) { + /* keep track of damaged files */ + for(j=0;jdisk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block)); @@ -456,6 +465,9 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size); if (ret == -1) { + /* mark the file as damaged */ + file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED); + if (errno == EACCES) { fprintf(stderr, "WARNING! Please give write permission to the file.\n"); } else { @@ -467,6 +479,9 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity goto bail; } + /* mark the file as fixed */ + file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED); + fprintf(stderr, "fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block)); ++recovered_error; } @@ -510,6 +525,66 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } } + if (fix) { + /* for all the files of this block check if we need to fix the modification time */ + for(j=0;jinodeset, file_inode_compare, &inode, file_inode_hash(inode)); + + /* if the inode is in the database and it refers at a different file name, */ + /* we can fix the file time ONLY if the time and size allow to differentiates */ + /* between the two files */ + + /* for example, suppose we delete a bunch of files with all the same size and time, */ + /* when recreating them the inodes may be reused in a different order, */ + /* and at the next sync some files may have matching inode/size/time even if different name */ + /* not allowing sync to detect that the file is changed and not renamed */ + if (!collide_file /* if not in the databse, there is no collision */ + || strcmp(collide_file->sub, handle[j].file->sub) == 0 /* if the name is the same, it's the right collision */ + || collide_file->size != handle[j].file->size /* if the size is different, the collision is identified */ + || collide_file->mtime != handle[j].file->mtime /* if the mtime is different, the collision is identified */ + ) { + /* set the original modification time */ + ret = handle_utime(&handle[j]); + if (ret == -1) { + /* mark the file as damaged */ + file_flag_set(handle[j].file, FILE_IS_DAMAGED); + fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); + printf("Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } + } else { + fprintf(stderr, "collision:%s:%s:%s: Not setting modification time to avoid inode collision\n", handle[j].disk->name, block_file_get(block)->sub, collide_file->sub); + } + } + } + /* count the number of processed block */ ++countpos; diff --git a/dry.c b/dry.c index ca01ac0d..4cc6c937 100644 --- a/dry.c +++ b/dry.c @@ -63,11 +63,14 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, int qar if (!block) continue; - ret = handle_close_if_different(&handle[j], block_file_get(block)); - if (ret == -1) { - printf("Stopping at block %u\n", i); - ++error; - goto bail; + /* if the file is different than the current one, close it */ + if (handle[j].file != block_file_get(block)) { + ret = handle_close(&handle[j]); + if (ret == -1) { + printf("Stopping at block %u\n", i); + ++error; + goto bail; + } } /* open the file for reading */ diff --git a/elem.c b/elem.c index ec278184..75768c3a 100644 --- a/elem.c +++ b/elem.c @@ -193,6 +193,13 @@ block_off_t block_file_pos(struct snapraid_block* block) return block - file->blockvec; } +int block_is_last(struct snapraid_block* block) +{ + struct snapraid_file* file = block_file_get(block); + + return block == file->blockvec + file->blockmax - 1; +} + unsigned block_file_size(struct snapraid_block* block, unsigned block_size) { block_off_t pos = block_file_pos(block); diff --git a/elem.h b/elem.h index b4775342..dde0a112 100644 --- a/elem.h +++ b/elem.h @@ -67,6 +67,7 @@ struct snapraid_file; #define BLOCK_HAS_HASH 1 /**< If the hash value is valid. */ #define BLOCK_HAS_PARITY 2 /**< If the parity block is valid. */ +#define BLOCK_HAS_MASK 3 /**< Mask of all the flags. Used to mix them inside the file pointer. */ /** * Block of a file. @@ -80,6 +81,8 @@ struct snapraid_block { #define FILE_IS_PRESENT 1 /**< If it's seen as present. */ #define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */ #define FILE_IS_LARGER 4 /**< If a larger file was already detected. */ +#define FILE_IS_DAMAGED 8 /**< If a fix was attempted but it failed. */ +#define FILE_IS_FIXED 16 /**< If a fix was done. */ /** * File. @@ -152,14 +155,20 @@ void filter_free(struct snapraid_filter* filter); */ int filter_path(tommy_list* filterlist, const char* path, int is_dir); +/** + * Gets the file containing the block. + */ static inline struct snapraid_file* block_file_get(struct snapraid_block* block) { - return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)3); + return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)BLOCK_HAS_MASK); } +/** + * Sets the file containing the block. + */ static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file) { - block->file_mixed = (block->file_mixed & ~(uintptr_t)3) | (uintptr_t)file; + block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_HAS_MASK) | (uintptr_t)file; } static inline int block_flag_has(struct snapraid_block* block, unsigned mask) @@ -182,6 +191,11 @@ static inline void block_flag_clear(struct snapraid_block* block, unsigned mask) */ block_off_t block_file_pos(struct snapraid_block* block); +/** + * Checks if the block is the last in the file. + */ +int block_is_last(struct snapraid_block* block); + /** * Gets the size in bytes of the block. * If it's the last block of a file it could be less than block_size. diff --git a/handle.c b/handle.c index 9fbe1fcc..deedd289 100644 --- a/handle.c +++ b/handle.c @@ -226,16 +226,6 @@ int handle_close(struct snapraid_handle* handle) return 0; } -int handle_close_if_different(struct snapraid_handle* handle, struct snapraid_file* file) -{ - /* if it's the same file, nothing to do */ - if (handle->file == file) { - return 0; - } - - return handle_close(handle); -} - int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size) { ssize_t read_ret; @@ -309,4 +299,27 @@ int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, u return 0; } +int handle_utime(struct snapraid_handle* handle) +{ + struct timeval tv[2]; + int ret; + + /* do nothing if not opened */ + if (handle->f == -1) + return 0; + + tv[0].tv_sec = handle->file->mtime; + tv[0].tv_usec = 0; + + tv[1].tv_sec = tv[0].tv_sec; + tv[1].tv_usec = tv[0].tv_usec; + + ret = futimes(handle->f, tv); + if (ret != 0) { + fprintf(stderr, "Error timing file '%s'. %s.\n", handle->file->sub, strerror(errno)); + return -1; + } + + return 0; +} diff --git a/handle.h b/handle.h index b3e6d41c..2e5d2be2 100644 --- a/handle.h +++ b/handle.h @@ -34,11 +34,6 @@ struct snapraid_handle { */ int handle_ancestor(const char* file); -/** - * Closes a file, if it's different from the specified one. - */ -int handle_close_if_different(struct snapraid_handle* handle, struct snapraid_file* file); - /** * Creates a file. * The file is created if missing, and opened with write access. @@ -72,5 +67,10 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un */ int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size); +/** + * Changes the modification time of the file to the saved value. + */ +int handle_utime(struct snapraid_handle* handle); + #endif diff --git a/mingw.c b/mingw.c index 90b93053..b02fde74 100644 --- a/mingw.c +++ b/mingw.c @@ -318,6 +318,45 @@ int windows_ftruncate(int fd, off64_t off) return 0; } +int windows_futimes(int fd, struct timeval tv[2]) +{ + HANDLE h; + FILETIME ft; + uint64_t mtime; + + if (fd == -1) { + errno = EBADF; + return -1; + } + + h = (HANDLE)_get_osfhandle(fd); + if (h == INVALID_HANDLE_VALUE) { + errno = EBADF; + return -1; + } + + /* + * Convert to windows time + * + * How To Convert a UNIX time_t to a Win32 FILETIME or SYSTEMTIME + * http://support.microsoft.com/kb/167296 + */ + mtime = tv[0].tv_sec; + mtime *= 10000000; + mtime += tv[0].tv_usec * 10; + mtime += 116444736000000000; + + ft.dwHighDateTime = mtime >> 32; + ft.dwLowDateTime = mtime; + + if (!SetFileTime(h, 0, 0, &ft)) { + windows_errno(GetLastError()); + return -1; + } + + return 0; +} + int windows_rename(const char* a, const char* b) { /* diff --git a/mingw.h b/mingw.h index 97d1eee8..6e4e451d 100644 --- a/mingw.h +++ b/mingw.h @@ -49,6 +49,7 @@ #define opendir windows_opendir #define readdir windows_readdir #define closedir windows_closedir +#define futimes windows_futimes #define O_NOFOLLOW 0 #define S_ISLNK(mode) 0 #define readlink(a,b,c) -1 @@ -66,12 +67,12 @@ struct windows_stat { }; /** - * Like the C fstat() but without inode information. + * Like the C fstat() including the inode information. */ int windows_fstat(int fd, struct windows_stat* st); /** - * Like the C stat() but without inode information. + * Like the C stat() but without the inode information. */ int windows_lstat(const char* file, struct windows_stat* st); @@ -86,7 +87,7 @@ int windows_access(const char* file, int mode); int windows_mkdir(const char* file); /** - * Like the C lstat() with inode information. + * Like the C lstat() including the inode information. * It doesn't work for all kind of files and directories. For example "\System Volume Information" cannot be opened. * Note that instead lstat() works for all the files. */ @@ -98,6 +99,11 @@ int lstat_inode(const char* file, struct windows_stat* st); */ int windows_ftruncate(int fd, off64_t off); +/** + * Like the C futimes(). + */ +int windows_futimes(int fd, struct timeval tv[2]); + /** * Like the C rename(). */ diff --git a/snapraid.1 b/snapraid.1 index ac7727f8..4359c2a1 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -101,8 +101,8 @@ It\(cqs able to recover damages only from up to two disks. With a Backup you are able to recover from a complete failure of the whole disk array. .IP \(bu -Only file data and symlinks are saved. Permissions, time, extended attributes, -hard\(hylinks are not saved. +Only file data, time and symlinks are saved. Permissions, +extended attributes and hard\(hylinks are not saved. .PD .SH GETTING STARTED To use SnapRAID you need first select one disk of your disk array diff --git a/snapraid.d b/snapraid.d index 0e854815..4699032b 100644 --- a/snapraid.d +++ b/snapraid.d @@ -67,8 +67,8 @@ Limitations * It's able to recover damages only from up to two disks. With a Backup you are able to recover from a complete failure of the whole disk array. - * Only file data and symlinks are saved. Permissions, time, extended attributes, - hard-links are not saved. + * Only file data, time and symlinks are saved. Permissions, + extended attributes and hard-links are not saved. Getting Started To use SnapRAID you need first select one disk of your disk array diff --git a/snapraid.txt b/snapraid.txt index 0e0dd05c..6056eff5 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -77,8 +77,8 @@ Other limitations are: * It's able to recover damages only from up to two disks. With a Backup you are able to recover from a complete failure of the whole disk array. -* Only file data and symlinks are saved. Permissions, time, extended attributes, - hard-links are not saved. +* Only file data, time and symlinks are saved. Permissions, + extended attributes and hard-links are not saved. 4 GETTING STARTED diff --git a/sync.c b/sync.c index c933873b..50f96694 100644 --- a/sync.c +++ b/sync.c @@ -116,14 +116,17 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa continue; } - ret = handle_close_if_different(&handle[j], block_file_get(block)); - if (ret == -1) { - /* This one is really an unexpected error, because we are only reading */ - /* and closing a descriptor should never fail */ - fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n"); - printf("Stopping at block %u\n", i); - ++unrecoverable_error; - goto bail; + /* if the file is different than the current one, close it */ + if (handle[j].file != block_file_get(block)) { + ret = handle_close(&handle[j]); + if (ret == -1) { + /* This one is really an unexpected error, because we are only reading */ + /* and closing a descriptor should never fail */ + fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to sync.\n"); + printf("Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } } ret = handle_open(&handle[j], block_file_get(block));