diff --git a/HISTORY b/HISTORY index 6e175b37..48cb5f62 100644 --- a/HISTORY +++ b/HISTORY @@ -4,6 +4,7 @@ SnapRAID HISTORY 1.8 2011/12 =========== * Added a new "dup" command to find all the duplicate files. + * Added a new option "--filter-nohidden" to exclude hidden files. * The example configuration files now put the content files in the data disks instead than in the parity disks. diff --git a/TODO b/TODO index b1a8c352..18b2a4bb 100644 --- a/TODO +++ b/TODO @@ -19,21 +19,16 @@ with an atomic rename operation. * Recheck the 2nd issue reported at: https://sourceforge.net/projects/snapraid/forums/forum/1677233/topic/4747897 -* If a directory exist with the same name of a parity/content file be more explicative +* If a directory exists with the same name of a parity/content file be more explicative on the error message. See: https://sourceforge.net/projects/snapraid/forums/forum/1677233/topic/4861034 Also adding an estension to these files in the configuration file for Windows may help to understand that they are files and not directories. -* Put the content files in the data disks in the default configuration file. - * Easy way to find files using a specified block. * Adds a checksum for the content file. When reading if the file is invalid, skip to another. -* Add an option to exclude "hidden" file. In Windows files with the HIDDEN attributes, -in Unix files starting with '.'. - - v1.9 * Allow to filter the files to check/fix based on the drive where they reside. diff --git a/elem.h b/elem.h index 3bdec883..f988f812 100644 --- a/elem.h +++ b/elem.h @@ -166,23 +166,36 @@ struct snapraid_filter* filter_alloc(int is_include, const char* pattern); */ void filter_free(struct snapraid_filter* filter); +/** + * Filter hidden files. + * Return !=0 if it matches and it should be excluded. + */ +static inline int filter_hidden(int enable, struct dirent* dd, struct stat* st) +{ + if (enable && ishidden(dd, st)) { + return 1; /* filter out */ + } + + return 0; +} + /** * Filters a path using a list of filters. * For each element of the path all the filters are applied, until the first one that matches. - * Return !=0 if it matches. + * Return !=0 if it matches and it should be excluded. */ int filter_path(tommy_list* filterlist, const char* sub); /** * Filters a dir using a list of filters. * For each element of the path all the filters are applied, until the first one that matches. - * Return !=0 if it matches. + * Return !=0 if it matches and it should be excluded. */ int filter_dir(tommy_list* filterlist, const char* sub); /** * Filters a path if it's a content file. - * Return !=0 if it matches. + * Return !=0 if it matches and it should be excluded. */ int filter_content(tommy_list* contentlist, const char* path); diff --git a/mingw.c b/mingw.c index cb7b8e98..b6048eba 100644 --- a/mingw.c +++ b/mingw.c @@ -83,8 +83,6 @@ static char* u16tou8(const wchar_t* src) static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, struct windows_stat* st) { /* Convert special attributes to a char device */ - /* Note that the FILE_ATTRIBUTE_HIDDEN attribute is intentionally ignored as */ - /* hidden files should be backuped like any other */ if ((info->dwFileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { st->st_mode = S_IFBLK; } else if ((info->dwFileAttributes & ( @@ -100,6 +98,9 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, struct win st->st_mode = S_IFREG; } + /* Store the HIDDEN attribute in a separate field */ + st->st_ishidden = (info->dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0; + st->st_size = info->nFileSizeHigh; st->st_size <<= 32; st->st_size |= info->nFileSizeLow; @@ -131,15 +132,13 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, struct win static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_stat* st) { /* Convert special attributes */ - /* Note that the FILE_ATTRIBUTE_HIDDEN attribute is intentionally ignored as */ - /* hidden files should be backuped like any other */ if ((info->dwFileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { st->st_mode = S_IFBLK; } else if ((info->dwFileAttributes & ( FILE_ATTRIBUTE_SYSTEM /* System files */ | FILE_ATTRIBUTE_TEMPORARY /* Files going to be deleted on close */ | FILE_ATTRIBUTE_OFFLINE - | FILE_ATTRIBUTE_REPARSE_POINT /* Symbolic links */ + | FILE_ATTRIBUTE_REPARSE_POINT /* Symbolic links, not supported so return a character file */ )) != 0) { st->st_mode = S_IFCHR; } else if ((info->dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { @@ -148,6 +147,9 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s st->st_mode = S_IFREG; } + /* Store the HIDDEN attribute in a separate field */ + st->st_ishidden = (info->dwFileAttributes & FILE_ATTRIBUTE_HIDDEN) != 0; + st->st_size = info->nFileSizeHigh; st->st_size <<= 32; st->st_size |= info->nFileSizeLow; @@ -535,5 +537,11 @@ int windows_closedir(windows_dir* dirstream) return 0; } +int windows_ishidden(struct dirent* dd, struct windows_stat* st) +{ + (void)dd; + return st->st_ishidden; +} + #endif diff --git a/mingw.h b/mingw.h index 63b4a5de..5309bd72 100644 --- a/mingw.h +++ b/mingw.h @@ -51,6 +51,8 @@ #define closedir windows_closedir #define futimes windows_futimes #define O_NOFOLLOW 0 +#define ishidden windows_ishidden +/* In Windows symbolic links are not supported */ #define S_ISLNK(mode) 0 #define readlink(a,b,c) -1 #define symlink(a,b) -1 @@ -65,6 +67,7 @@ struct windows_stat { uint32_t st_mode; uint32_t st_nlink; uint32_t st_dev; + int st_ishidden; }; /** @@ -163,6 +166,11 @@ struct windows_dirent* windows_readdir(windows_dir* dirstream); */ int windows_closedir(windows_dir* dirstream); +/** + * Like ishidden(). + */ +int windows_ishidden(struct dirent* dd, struct windows_stat* st); + #endif #endif diff --git a/portable.h b/portable.h index f60cf95d..a700f34c 100644 --- a/portable.h +++ b/portable.h @@ -110,13 +110,20 @@ #define SWITCH_GETOPT_LONG(a,b) b #endif -#ifndef __MINGW32__ /* Specific for Unix */ -#define O_BINARY 0 -#define O_SEQUENTIAL 0 -#endif - #ifdef __MINGW32__ /* Specific for MingW */ #include "mingw.h" +#else /* Specific for Unix */ +#define O_BINARY 0 /**< Not used in Unix. */ +#define O_SEQUENTIAL 0 /**< In Unix posix_fadvise() shall be used. */ + +/** + * Check if the specified file is hidden. + */ +static inline int ishidden(struct dirent* dd, struct stat* st) +{ + (void)st; + return dd->d_name[0] == '.'; +} #endif #endif diff --git a/scan.c b/scan.c index 2a0d71c9..6e6670e7 100644 --- a/scan.c +++ b/scan.c @@ -420,7 +420,8 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } if (S_ISREG(st.st_mode)) { - if (filter_content(&state->contentlist, path_next) == 0 + if (filter_hidden(state->filter_hidden, dd, &st) == 0 + && filter_content(&state->contentlist, path_next) == 0 && filter_path(&state->filterlist, sub_next) == 0 ) { /* check for read permission */ @@ -444,7 +445,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } } } else if (S_ISLNK(st.st_mode)) { - if (filter_path(&state->filterlist, sub_next) == 0) { + if (filter_hidden(state->filter_hidden, dd, &st) == 0 + && filter_path(&state->filterlist, sub_next) == 0 + ) { char subnew[PATH_MAX]; int ret; @@ -468,7 +471,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } } } else if (S_ISDIR(st.st_mode)) { - if (filter_dir(&state->filterlist, sub_next) == 0) { + if (filter_hidden(state->filter_hidden, dd, &st) == 0 + && filter_dir(&state->filterlist, sub_next) == 0 + ) { pathslash(path_next, sizeof(path_next)); pathslash(sub_next, sizeof(sub_next)); scan_dir(scan, state, output, disk, path_next, sub_next); @@ -478,7 +483,8 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } } } else { - if (filter_content(&state->contentlist, path_next) == 0 + if (filter_hidden(state->filter_hidden, dd, &st) == 0 + && filter_content(&state->contentlist, path_next) == 0 && filter_path(&state->filterlist, sub_next) == 0 ) { fprintf(stderr, "warning: Ignoring special file '%s'\n", path_next); diff --git a/snapraid.1 b/snapraid.1 index 23d0caf6..3780006d 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -2,7 +2,7 @@ .SH NAME snapraid \(hy SnapRAID Backup For Disk Arrays .SH SYNOPSIS -snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hyf, \(hy\(hyfilter PATTERN] +snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hyf, \(hy\(hyfilter PATTERN] [\(hyH, \(hy\(hyfilter\(hynohidden] .PD 0 .PP .PD @@ -352,6 +352,11 @@ pattern specifications. This option can be used many times. In Unix, ensure to quote globbing chars if used. .TP +.B \(hyH, \(hy\(hyfilter\(hynohidden +Filter out hidden files and directory. In Unix hidden files are +the ones starting with \(cq.\(cq. In Windows they are the ones with the +hidden attribute. +.TP .B \(hyN, \(hy\(hyfind\(hyby\(hyname When syncing finds the files by path instead than by inode. This option allows a fast sync command after having replaced @@ -452,6 +457,9 @@ You can change the mount point as you like, as long you keep the NAME fixed. .PP You should use one option for each disk of the array. +.SS nohidden +Excludes all the hidden files and directory, like the +\(hy\(hyfilter\(hynohidden option. .SS exclude/include PATTERN Defines the file or directory patterns to exclude and include in the sync process. diff --git a/snapraid.c b/snapraid.c index 60acb4ba..991f3762 100644 --- a/snapraid.c +++ b/snapraid.c @@ -76,6 +76,7 @@ void usage(void) struct option long_options[] = { { "conf", 1, 0, 'c' }, { "filter", 1, 0, 'f' }, + { "filter-nohidden", 0, 0, 'H' }, { "start", 1, 0, 's' }, { "count", 1, 0, 't' }, { "force-zero", 0, 0, 'Z' }, @@ -96,7 +97,7 @@ struct option long_options[] = { }; #endif -#define OPTIONS "c:f:s:t:ZENTvhVG" +#define OPTIONS "c:f:Hs:t:ZENTvhVG" volatile int global_interrupt = 0; @@ -124,6 +125,7 @@ int main(int argc, char* argv[]) int force_zero; int force_empty; int find_by_name; + int filter_hidden; int test_expect_unrecoverable; int test_expect_recoverable; int test_kill_after_sync; @@ -145,6 +147,7 @@ int main(int argc, char* argv[]) force_zero = 0; force_empty = 0; find_by_name = 0; + filter_hidden = 0; test_expect_unrecoverable = 0; test_expect_recoverable = 0; test_kill_after_sync = 0; @@ -174,6 +177,9 @@ int main(int argc, char* argv[]) } tommy_list_insert_tail(&filterlist, &filter->node, filter); } break; + case 'H' : + filter_hidden = 1; + break; case 's' : blockstart = strtoul(optarg, &e, 0); if (!e || *e) { @@ -262,7 +268,7 @@ int main(int argc, char* argv[]) state_init(&state); - state_config(&state, conf, verbose, gui, force_zero, force_empty, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_device); + state_config(&state, conf, verbose, gui, force_zero, force_empty, filter_hidden, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_device); if (operation == OPERATION_DIFF) { state_read(&state); diff --git a/snapraid.conf.example b/snapraid.conf.example index 12c61305..b9367f28 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -28,6 +28,9 @@ disk d1 /mnt/disk1/ disk d2 /mnt/disk2/ disk d3 /mnt/disk3/ +# Excludes hidden files and directories (uncomment to enable). +# nohidden + # Defines files and directories to exclude # Remember that all the paths are relative at the mount points # Format: "exclude FILE" diff --git a/snapraid.conf.example.windows b/snapraid.conf.example.windows index 7c921e2d..48488026 100644 --- a/snapraid.conf.example.windows +++ b/snapraid.conf.example.windows @@ -28,6 +28,9 @@ disk d1 G:\array\ disk d2 H:\array\ disk d3 I:\array\ +# Excludes hidden files and directories (uncomment to enable). +# nohidden + # Defines files and directories to exclude # Remember that all the paths are relative at the mount points # Format: "exclude FILE" diff --git a/snapraid.d b/snapraid.d index 5cc4aa5e..b512fb9a 100644 --- a/snapraid.d +++ b/snapraid.d @@ -2,7 +2,7 @@ Name{number} snapraid - SnapRAID Backup For Disk Arrays Synopsis - :snapraid [-c, --conf CONFIG] [-f, --filter PATTERN] + :snapraid [-c, --conf CONFIG] [-f, --filter PATTERN] [-H, --filter-nohidden] : [-N, --find-by-name] : [-Z, --force-zero] [-E, --force-empty] : [-s, --start BLKSTART] [-t, --count BLKCOUNT] @@ -237,6 +237,11 @@ Options This option can be used many times. In Unix, ensure to quote globbing chars if used. + -H, --filter-nohidden + Filter out hidden files and directory. In Unix hidden files are + the ones starting with '.'. In Windows they are the ones with the + hidden attribute. + -N, --find-by-name When syncing finds the files by path instead than by inode. This option allows a fast sync command after having replaced @@ -343,6 +348,10 @@ Configuration You should use one option for each disk of the array. + nohidden + Excludes all the hidden files and directory, like the + --filter-nohidden option. + exclude/include PATTERN Defines the file or directory patterns to exclude and include in the sync process. diff --git a/snapraid.txt b/snapraid.txt index e3d500a1..37ec044a 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -6,7 +6,7 @@ 1 SYNOPSIS ========== -snapraid [-c, --conf CONFIG] [-f, --filter PATTERN] +snapraid [-c, --conf CONFIG] [-f, --filter PATTERN] [-H, --filter-nohidden] [-N, --find-by-name] [-Z, --force-zero] [-E, --force-empty] [-s, --start BLKSTART] [-t, --count BLKCOUNT] @@ -266,6 +266,11 @@ SnapRAID provides the following options: This option can be used many times. In Unix, ensure to quote globbing chars if used. + -H, --filter-nohidden + Filter out hidden files and directory. In Unix hidden files are + the ones starting with '.'. In Windows they are the ones with the + hidden attribute. + -N, --find-by-name When syncing finds the files by path instead than by inode. This option allows a fast sync command after having replaced @@ -383,7 +388,13 @@ keep the NAME fixed. You should use one option for each disk of the array. -7.5 exclude/include PATTERN +7.5 nohidden +------------ + +Excludes all the hidden files and directory, like the +--filter-nohidden option. + +7.6 exclude/include PATTERN --------------------------- Defines the file or directory patterns to exclude and include @@ -401,7 +412,7 @@ specifications. This option can be used many times. -7.6 block_size SIZE_IN_KIBIBYTES +7.7 block_size SIZE_IN_KIBIBYTES -------------------------------- Defines the basic block size in kibi bytes of the redundancy @@ -432,7 +443,7 @@ wasted for each file. For example, with 10000 files and a 256 KiB block size, you are going to waste 1.2 GiB. -7.7 Examples +7.8 Examples ------------ An example of a typical configuration for Unix is: diff --git a/state.c b/state.c index 2f2130a8..a50eefac 100644 --- a/state.c +++ b/state.c @@ -27,6 +27,7 @@ void state_init(struct snapraid_state* state) state->gui = 0; state->force_zero = 0; state->force_empty = 0; + state->filter_hidden = 0; state->find_by_name = 0; state->expect_unrecoverable = 0; state->expect_recoverable = 0; @@ -130,7 +131,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, i } } -void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_device) +void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int filter_hidden, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_device) { STREAM* f; unsigned line; @@ -139,6 +140,7 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i state->gui = gui; state->force_zero = force_zero; state->force_empty = force_empty; + state->filter_hidden = filter_hidden; state->find_by_name = find_by_name; state->expect_unrecoverable = expect_unrecoverable; state->expect_recoverable = expect_recoverable; @@ -323,6 +325,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i disk = disk_alloc(buffer, dir, st.st_dev); tommy_list_insert_tail(&state->disklist, &disk->node, disk); + } else if (strcmp(tag, "nohidden") == 0) { + state->filter_hidden = 1; } else if (strcmp(tag, "exclude") == 0) { struct snapraid_filter* filter; diff --git a/state.h b/state.h index 69db8861..514c23c3 100644 --- a/state.h +++ b/state.h @@ -34,6 +34,7 @@ struct snapraid_state { int gui; /**< Gui output. */ int force_zero; /**< Forced dangerous operations of synching file now with zero size. */ int force_empty; /**< Forced dangerous operations of synching disk now empty. */ + int filter_hidden; /**< Filter out hidden files. */ int find_by_name; /**< Forced dangerous operations of synching a rewritten disk. */ int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */ int expect_recoverable; /**< Expect presence of recoverable error in checking. */ @@ -66,7 +67,7 @@ void state_done(struct snapraid_state* state); /** * Read the configuration file. */ -void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_device); +void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int filter_hidden, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_device); /** * Read the state.