From b9b8fd0bb8e06b2cfd8f4c6c6ecf9746bc643469 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 16 Mar 2011 23:52:10 +0100 Subject: [PATCH] Adds filter exclusion. Some refactoring. --- HISTORY | 10 ++- Makefile.am | 10 +-- README | 11 ++- check.c | 2 +- config.h | 3 + configure.ac | 2 +- elem.c | 92 +++++++++++++++++--- elem.h | 28 +++++- handle.c | 5 +- portable.h | 4 + scan.c | 67 +++++++++++---- snapraid.c | 2 +- snapraid.conf.example | 19 +++-- snapraid.d | 56 ++++++++++-- snapraid.html | 87 +++++++++++++++++-- snapraid.txt | 67 +++++++++++++-- state.c | 22 +++-- state.h | 3 +- sync.c | 2 +- test.conf | 6 +- tommylist.h | 26 ++++++ tommytypes.h | 11 +++ util.c | 192 ++++++++++++++++++++++++++++++++++++++++++ util.h | 177 ++++++++++---------------------------- 24 files changed, 687 insertions(+), 217 deletions(-) diff --git a/HISTORY b/HISTORY index e249f2d1..fe35845f 100644 --- a/HISTORY +++ b/HISTORY @@ -1,8 +1,14 @@ SnapRAID HISTORY ================ +0.2 2011/3 +========== +* Exclusion list of files and directories +* More automatic tests +* Better documentation + + 0.1 2011/3 ========== - -* First public release of SnapRAID. +* First public test release of SnapRAID. diff --git a/Makefile.am b/Makefile.am index 9f689613..549301f3 100644 --- a/Makefile.am +++ b/Makefile.am @@ -52,9 +52,10 @@ clean-local: rm -rf test maintainer-clean-local: - rm -f README AUTHORS HISTORY INSTALL rm -f snapraid.1 snapraid.txt snapraid.html +# Automatic testing + CHECK = ./snapraid -v -c test.conf fix check-local: @@ -90,9 +91,11 @@ check-local: ./mktest damage 1 1 500 test/disk2/b* ./snapraid -c $(srcdir)/test.conf check 2>/dev/null ./snapraid -c $(srcdir)/test.conf fix 2>/dev/null - ./snapraid -c $(srcdir)/test.conf check + ./snapraid -c $(srcdir)/test.conf check echo Success! +# Valgrind testing + valgrind: valgrind \ --tool=memcheck \ @@ -130,9 +133,6 @@ cachegrind: %.html : %.d advd2 html < $(srcdir)/$< > $@ -%.hh : %.d - advd2 frame < $(srcdir)/$< > $@ - # Windows distribution DISTWIN = \ diff --git a/README b/README index a529d906..940c6a31 100644 --- a/README +++ b/README @@ -12,15 +12,14 @@ a lot of big files that rarely change. The main features of SnapRAID are: * You can start using SnapRAID with already filled disks. - You do not have to reformat or copy your data. * The disks of the array can have different sizes. - You have only to use the biggest one for the redundancy data. +* You can add more disks at the array at any time. * If you accidentally delete some files in a disk, you can recover them. -* If more than one disk fails, you lose the data only on the failed disks. - All the data in the other disks is safe. -* It doesn't lock-in your data. - You can stop using SnapRAID at any time. +* If more than one disk fails, you lose the data only on the + failed disks. All the data in the other disks is safe. +* It doesn't lock-in your data. You can stop using SnapRAID + at any time without the need to reformat or move data. The official site of SnapRAID is: diff --git a/check.c b/check.c index c9ece0a5..2c176511 100644 --- a/check.c +++ b/check.c @@ -292,7 +292,7 @@ void state_check(struct snapraid_state* state, int fix, block_off_t blockstart) exit(EXIT_FAILURE); } - snprintf(path, sizeof(path), "%s", state->parity); + pathcpy(path, sizeof(path), state->parity); if (fix) { /* if fixing, create the file and open for writing */ /* it never fails */ diff --git a/config.h b/config.h index b3e6c060..89dbb1f2 100644 --- a/config.h +++ b/config.h @@ -17,6 +17,9 @@ /* Define to 1 if you have the `fgetc_unlocked' function. */ #define HAVE_FGETC_UNLOCKED 1 +/* Define to 1 if you have the header file. */ +#define HAVE_FNMATCH_H 1 + /* Define to 1 if you have the `fsync' function. */ #define HAVE_FSYNC 1 diff --git a/configure.ac b/configure.ac index 61181f04..c0129dfe 100644 --- a/configure.ac +++ b/configure.ac @@ -18,7 +18,7 @@ AC_HEADER_ASSERT AC_HEADER_STDC AC_HEADER_DIRENT AC_HEADER_TIME -AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h sys/time.h unistd.h getopt.h]) +AC_CHECK_HEADERS([fcntl.h stddef.h stdint.h stdlib.h string.h sys/time.h unistd.h getopt.h fnmatch.h]) dnl Checks for typedefs, structures, and compiler characteristics. AC_C_CONST diff --git a/elem.c b/elem.c index 9ba9106e..847a7104 100644 --- a/elem.c +++ b/elem.c @@ -23,6 +23,81 @@ /****************************************************************************/ /* snapraid */ +struct snapraid_filter* filter_alloc(const char* pattern) +{ + struct snapraid_filter* filter; + char* i; + char* first; + char* last; + + filter = malloc_nofail(sizeof(struct snapraid_filter)); + pathcpy(filter->pattern, sizeof(filter->pattern), pattern); + + /* find first and last slash */ + first = 0; + last = 0; + for(i=filter->pattern;*i;++i) { + if (*i == '/') { + if (!first) + first = i; + last = i; + } + } + + if (first == 0) { + /* no slash */ + filter->is_path = 0; + filter->is_dir = 0; + } else if (first == last && last[1] == 0) { + /* one slash at the end */ + filter->is_path = 0; + filter->is_dir = 1; + last[0] = 0; + } else { + /* at least a slash not at the end */ + filter->is_path = 1; + if (last[1] == 0) { + filter->is_dir = 1; + last[0] = 0; + } else { + filter->is_dir = 0; + } + + /* a slash must be the first char, as we reject PATH/DIR/FILE */ + if (filter->pattern[0] != '/') { + free(filter); + return 0; + } + } + + return filter; +} + +void filter_free(struct snapraid_filter* filter) +{ + free(filter); +} + +int filter_filter(struct snapraid_filter* filter, const char* path, const char* name, int is_dir) +{ + /* matches dirs with dirs and files with files */ + if (filter->is_dir && !is_dir) + return -1; + if (!filter->is_dir && is_dir) + return -1; + + if (filter->is_path) { + /* skip initial slash, as always missing from the path */ + if (fnmatch(filter->pattern + 1, path, FNM_PATHNAME) == 0) + return 0; + } else { + if (fnmatch(filter->pattern, name, 0) == 0) + return 0; + } + + return -1; +} + block_off_t block_file_pos(struct snapraid_block* block) { struct snapraid_file* file = block->file; @@ -59,7 +134,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ block_off_t i; file = malloc_nofail(sizeof(struct snapraid_file)); - snprintf(file->sub, sizeof(file->sub), "%s", sub); + pathcpy(file->sub, sizeof(file->sub), sub); file->size = size; file->blockmax = (size + block_size - 1) / block_size; file->mtime = mtime; @@ -94,20 +169,11 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir) struct snapraid_disk* disk; disk = malloc_nofail(sizeof(struct snapraid_disk)); - snprintf(disk->name, sizeof(disk->name), "%s", name); - snprintf(disk->dir, sizeof(disk->dir), "%s", dir); + pathcpy(disk->name, sizeof(disk->name), name); + pathcpy(disk->dir, sizeof(disk->dir), dir); /* ensure that the dir terminate with "/" if it isn't empty */ - if (disk->dir[0] != 0) { - char last = disk->dir[strlen(disk->dir) - 1]; - if (last != '/' -#ifdef _WIN32 - && last != '\\' -#endif - ) { - snprintf(disk->dir, sizeof(disk->dir), "%s/", dir); - } - } + pathslash(disk->dir, sizeof(disk->dir)); disk->first_free_block = 0; tommy_list_init(&disk->filelist); diff --git a/elem.h b/elem.h index b52c128a..fd1f5f4c 100644 --- a/elem.h +++ b/elem.h @@ -48,6 +48,16 @@ typedef uint32_t block_off_t; */ typedef uint64_t data_off_t; +/** + * Filter for paths. + */ +struct snapraid_filter { + char pattern[PATH_MAX]; /**< Filter pattern. */ + int is_path; /**< If the pattern is only for the complete path. */ + int is_dir; /**< If the pattern is only for dir. */ + tommy_node node; /**< Next node in the list. */ +}; + struct snapraid_file; /** @@ -88,6 +98,22 @@ struct snapraid_disk { tommy_array blockarr; /**< Block array of the disk. */ }; +/** + * Allocates an exclusion. + */ +struct snapraid_filter* filter_alloc(const char* pattern); + +/** + * Deallocates an exclusion. + */ +void filter_free(struct snapraid_filter* filter); + +/** + * Checks if a path/name matches the pattern. + * Returns 0 if it matches. + */ +int filter_filter(struct snapraid_filter* filter, const char* path, const char* name, int is_dir); + /** * Gets the relative position of a block inside the file. */ @@ -105,7 +131,7 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size); struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, time_t mtime); /** - * Deallocate a file. + * Deallocates a file. */ void file_free(struct snapraid_file* file); diff --git a/handle.c b/handle.c index bdae52e2..3b3887d3 100644 --- a/handle.c +++ b/handle.c @@ -18,6 +18,7 @@ #include "portable.h" #include "elem.h" +#include "util.h" #include "handle.h" /****************************************************************************/ @@ -42,7 +43,7 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file) } } - snprintf(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub); + pathprint(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub); handle->f = open(handle->path, O_RDWR | O_CREAT | O_BINARY, 0600); if (handle->f == -1) { @@ -115,7 +116,7 @@ int handle_open(int ret_on_error, struct snapraid_handle* handle, struct snaprai } } - snprintf(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub); + pathprint(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub); handle->f = open(handle->path, O_RDONLY | O_BINARY); if (handle->f == -1) { diff --git a/portable.h b/portable.h index e12ba289..1cd1a81f 100644 --- a/portable.h +++ b/portable.h @@ -90,6 +90,10 @@ #include #endif +#if HAVE_FNMATCH_H +#include +#endif + #if HAVE_GETOPT_LONG #define SWITCH_GETOPT_LONG(a,b) a #else diff --git a/scan.c b/scan.c index b789559c..2c550134 100644 --- a/scan.c +++ b/scan.c @@ -29,7 +29,7 @@ struct snapraid_scan { }; /** - * Remove the specified file from the data set. + * Removes the specified file from the data set. */ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) { @@ -55,7 +55,7 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* } /** - * Insert the specified file in the data set. + * Inserts the specified file in the data set. */ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) { @@ -91,6 +91,9 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); } +/** + * Processes a file. + */ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* path, const char* sub, const struct stat* st) { struct snapraid_file* file; @@ -122,6 +125,25 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, scan_file_insert(state, disk, file); } +/** + * Checks if the specified path/file should be listet. + */ +static int scan_filter(struct snapraid_state* state, const char* path, const char* file, int is_dir) +{ + tommy_node* node = tommy_list_head(&state->excludelist); + while (node) { + struct snapraid_filter* filter = node->data; + if (filter_filter(filter, path, file, is_dir) == 0) + return -1; + node = node->next; + } + + return 0; +} + +/** + * Processes a directory. + */ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, struct snapraid_disk* disk, const char* dir, const char* sub) { DIR* d; @@ -134,32 +156,43 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s } while ((dd = readdir(d)) != 0) { - char path[PATH_MAX]; + char path_next[PATH_MAX]; + char sub_next[PATH_MAX]; + const char* name = dd->d_name; struct stat st; - snprintf(path, sizeof(path), "%s%s", dir, dd->d_name); + pathprint(path_next, sizeof(path_next), "%s%s", dir, name); + pathprint(sub_next, sizeof(sub_next), "%s%s", sub, name); - if (stat(path, &st) != 0) { - fprintf(stderr, "Error in stat file '%s'\n", path); + if (stat(path_next, &st) != 0) { + fprintf(stderr, "Error in stat file '%s'\n", path_next); exit(EXIT_FAILURE); } if (S_ISREG(st.st_mode)) { - char sub_next[PATH_MAX]; - snprintf(sub_next, sizeof(sub_next), "%s%s", sub, dd->d_name); - scan_file(scan, state, disk, path, sub_next, &st); + if (scan_filter(state, sub_next, name, 0) == 0) { + scan_file(scan, state, disk, path_next, sub_next, &st); + } else { + if (state->verbose) { + printf("warning: Excluded file '/%s'\n", sub_next); + } + } } else if (S_ISDIR(st.st_mode)) { - if (strcmp(dd->d_name, ".") != 0 && strcmp(dd->d_name, "..") != 0 && strcmp(dd->d_name, "lost+found") != 0 - ) { - char dir_next[PATH_MAX]; - char sub_next[PATH_MAX]; - snprintf(dir_next, sizeof(dir_next), "%s%s/", dir, dd->d_name); - snprintf(sub_next, sizeof(sub_next), "%s%s/", sub, dd->d_name); - scan_dir(scan, state, disk, dir_next, sub_next); + int is_auto = name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)); + if (!is_auto) { + if (scan_filter(state, sub_next, name, 1) == 0) { + pathslash(path_next, sizeof(path_next)); + pathslash(sub_next, sizeof(sub_next)); + scan_dir(scan, state, disk, path_next, sub_next); + } else { + if (state->verbose) { + printf("warning: Excluded directory '/%s'\n", sub_next); + } + } } } else { if (state->verbose) { - printf("warning: File '%s/%s' not processed\n", dir, dd->d_name); + printf("warning: Ignored file '/%s'\n", sub_next); } } } diff --git a/snapraid.c b/snapraid.c index b4ef02b5..ce64c2b0 100644 --- a/snapraid.c +++ b/snapraid.c @@ -103,7 +103,7 @@ int main(int argc, char* argv[]) conf = optarg; break; case 's' : - if (stru(optarg, &blockstart) != 0) { + if (stru32(optarg, &blockstart) != 0) { fprintf(stderr, "Invalid start position '%s'\n", optarg); exit(EXIT_FAILURE); } diff --git a/snapraid.conf.example b/snapraid.conf.example index 34941a13..ed6c8e7c 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -1,10 +1,5 @@ # Configuration for snapraid -# Define the block size in kilo bytes (1024 bytes). -# Format: "block_size SIZE_IN_KB" -# Default value is 128 -> 128 kilo bytes -block_size 128 - # Define the file to use as parity storage # It must me in an different disk than the data disk # Format: "parity FILE_PATH" @@ -25,3 +20,17 @@ disk d4 /mnt/d4/ disk d5 /mnt/d5/ disk d6 /mnt/d6/ +# Define files and directories to exclude +# Format: "exclude FILE" +# Format: "exclude DIR/" +# Format: "exclude /PATH/FILE" +# Format: "exclude /PATH/DIR/" +exclude *.bak +exclude tmp/ +exclude /lost+found/ + +# Define the block size in kilo bytes (1024 bytes). +# Format: "block_size SIZE_IN_KB" +# Default value is 128 -> 128 kilo bytes +block_size 128 + diff --git a/snapraid.d b/snapraid.d index 2e9e48cc..b41a03ba 100644 --- a/snapraid.d +++ b/snapraid.d @@ -20,16 +20,14 @@ Description The main features of SnapRAID are: * You can start using SnapRAID with already filled disks. - You do not have to reformat or copy your data. * The disks of the array can have different sizes. - You have only to use the biggest one for the redundancy data. + * You can add more disks at the array at any time. * If you accidentally delete some files in a disk, you can recover them. * If more than one disk fails, you lose the data only on the failed disks. All the data in the other disks is safe. - * It doesn't lock-in your data. - You can stop using SnapRAID at any time without the - need to reformat or move data. + * It doesn't lock-in your data. You can stop using SnapRAID at any + time without the need to reformat or move data. The official site of SnapRAID is: @@ -148,6 +146,12 @@ Configuration keep the NAME fixed. You should use one option for each disk of the array. + =exclude GLOB_PATTERN + Defines the file or directory patterns to exclude from the sync + process. See the PATTERN section for more details in the + pattern specifications. + This option can be used many times. + =block_size SIZE_IN_KILOBYTES Defines the basic block size in kilo bytes of the redundancy blocks. The default is 128 and it should @@ -170,6 +174,9 @@ Configuration :disk d1 /mnt/disk1/ :disk d2 /mnt/disk2/ :disk d3 /mnt/disk3/ + :exclude *.bak + :exclude /lost+found/ + :exclude tmp/ :block_size 256 Commands @@ -196,6 +203,45 @@ Commands also tries to fix problems reverting the state of the disk array at the previous "sync" command. +Pattern + Patterns are used to define the files and directories to exclude + from the redundancy process. + + It makes sense to exclude any file not worth to be saved. + + There are four different types of patterns: + + =FILE + Excludes any file named as FILE. You can use any globbing + character like * and ?. + This pattern is applied only to files and not to directories. + + =DIR/ + Excludes any directory named DIR. You can use any globbing + character like * and ?. + This pattern is applied only to directories and not to files. + + =/PATH/FILE + Excludes the exact specified file path. You can use any + globbing character like * and ? but they never matches a + directory slash. + This pattern is applied only to files and not to directories. + + =/PATH/DIR/ + Excludes the exact specified directory path. You can use any + globbing character like * and ? but they never matches a + directory slash. + This pattern is applied only to directories and not to files. + + For example: + + :# Excludes any file named "*.bak" + :exclude *.bak + :# Excludes the root directory "lost+found" + :exclude /lost+found/ + :# Excludes any directory named "tmp" + :exclude tmp/ + Options -c, --conf CONFIG Selects the configuration file. If not specified is assumed diff --git a/snapraid.html b/snapraid.html index f603a6de..ac625d2d 100644 --- a/snapraid.html +++ b/snapraid.html @@ -31,11 +31,12 @@ The main features of SnapRAID are:
  • You can start using SnapRAID with already filled disks. -You do not have to reformat or copy your data.
  • The disks of the array can have different sizes. -You have only to use the biggest one for the redundancy data. +
    • +
  • +You can add more disks at the array at any time.
  • If you accidentally delete some files in a disk, you can @@ -46,9 +47,8 @@ If more than one disk fails, you lose the data only on the failed disks. All the data in the other disks is safe.
  • -It doesn't lock-in your data. -You can stop using SnapRAID at any time without the -need to reformat or move data. +It doesn't lock-in your data. You can stop using SnapRAID at any +time without the need to reformat or move data.

@@ -219,6 +219,16 @@ You should use one option for each disk of the array. +
+exclude GLOB_PATTERN +
+Defines the file or directory patterns to exclude from the sync +process. See the PATTERN section for more details in the +pattern specifications. +This option can be used many times. +
+ +
block_size SIZE_IN_KILOBYTES
Defines the basic block size in kilo bytes of @@ -252,6 +262,9 @@ content /mnt/diskpar/content disk d1 /mnt/disk1/ disk d2 /mnt/disk2/ disk d3 /mnt/disk3/ +exclude *.bak +exclude /lost+found/ +exclude tmp/ block_size 256
@@ -300,6 +313,66 @@ disk array at the previous "sync" command.

7 +Pattern +

+Patterns are used to define the files and directories to exclude +from the redundancy process. +

+It makes sense to exclude any file not worth to be saved. +

+There are four different types of patterns: + + +
+FILE +
+Excludes any file named as FILE. You can use any globbing +character like * and ?. +This pattern is applied only to files and not to directories. +
+ + +
+DIR/ +
+Excludes any directory named DIR. You can use any globbing +character like * and ?. +This pattern is applied only to directories and not to files. +
+ + +
+/PATH/FILE +
+Excludes the exact specified file path. You can use any +globbing character like * and ? but they never matches a +directory slash. +This pattern is applied only to files and not to directories. +
+ + +
+/PATH/DIR/ +
+Excludes the exact specified directory path. You can use any +globbing character like * and ? but they never matches a +directory slash. +This pattern is applied only to directories and not to files. +
+

+For example: +

+
+
+# Excludes any file named "*.bak"
+exclude *.bak
+# Excludes the root directory "lost+found"
+exclude /lost+found/
+# Excludes any directory named "tmp"
+exclude tmp/
+
+
+

8 Options

@@ -351,11 +424,11 @@ Prints a short help screen. Prints the program version.
-

8 +

9 Copyright

This file is Copyright (C) 2011 Andrea Mazzoleni -

9 +

10 See Also

rsync(1) diff --git a/snapraid.txt b/snapraid.txt index 1451a9e7..fb70cd74 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -27,16 +27,14 @@ a lot of big files that rarely change. The main features of SnapRAID are: * You can start using SnapRAID with already filled disks. - You do not have to reformat or copy your data. * The disks of the array can have different sizes. - You have only to use the biggest one for the redundancy data. +* You can add more disks at the array at any time. * If you accidentally delete some files in a disk, you can recover them. * If more than one disk fails, you lose the data only on the failed disks. All the data in the other disks is safe. -* It doesn't lock-in your data. - You can stop using SnapRAID at any time without the - need to reformat or move data. +* It doesn't lock-in your data. You can stop using SnapRAID at any + time without the need to reformat or move data. The official site of SnapRAID is: @@ -164,6 +162,12 @@ it should contains the following options: keep the NAME fixed. You should use one option for each disk of the array. + exclude GLOB_PATTERN + Defines the file or directory patterns to exclude from the sync + process. See the PATTERN section for more details in the + pattern specifications. + This option can be used many times. + block_size SIZE_IN_KILOBYTES Defines the basic block size in kilo bytes of the redundancy blocks. The default is 128 and it should @@ -186,6 +190,9 @@ An example of a typical configuration is: disk d1 /mnt/disk1/ disk d2 /mnt/disk2/ disk d3 /mnt/disk3/ + exclude *.bak + exclude /lost+found/ + exclude tmp/ block_size 256 @@ -216,7 +223,49 @@ SnapRAID provides three simple commands that allow to: disk array at the previous "sync" command. -7 OPTIONS +7 PATTERN +========= + +Patterns are used to define the files and directories to exclude +from the redundancy process. + +It makes sense to exclude any file not worth to be saved. + +There are four different types of patterns: + + FILE + Excludes any file named as FILE. You can use any globbing + character like * and ?. + This pattern is applied only to files and not to directories. + + DIR/ + Excludes any directory named DIR. You can use any globbing + character like * and ?. + This pattern is applied only to directories and not to files. + + /PATH/FILE + Excludes the exact specified file path. You can use any + globbing character like * and ? but they never matches a + directory slash. + This pattern is applied only to files and not to directories. + + /PATH/DIR/ + Excludes the exact specified directory path. You can use any + globbing character like * and ? but they never matches a + directory slash. + This pattern is applied only to directories and not to files. + +For example: + + # Excludes any file named "*.bak" + exclude *.bak + # Excludes the root directory "lost+found" + exclude /lost+found/ + # Excludes any directory named "tmp" + exclude tmp/ + + +8 OPTIONS ========= -c, --conf CONFIG @@ -245,13 +294,13 @@ SnapRAID provides three simple commands that allow to: Prints the program version. -8 COPYRIGHT +9 COPYRIGHT =========== This file is Copyright (C) 2011 Andrea Mazzoleni -9 SEE ALSO -========== +10 SEE ALSO +=========== rsync(1) diff --git a/state.c b/state.c index 721ac840..fd0a3259 100644 --- a/state.c +++ b/state.c @@ -33,6 +33,7 @@ void state_init(struct snapraid_state* state) state->content[0] = 0; state->parity[0] = 0; tommy_array_init(&state->diskarr); + tommy_list_init(&state->excludelist); if (sizeof(off_t) < sizeof(uint64_t)) { fprintf(stderr, "Missing support for large files\n"); @@ -43,9 +44,11 @@ void state_init(struct snapraid_state* state) void state_done(struct snapraid_state* state) { unsigned i; + for(i=0;idiskarr);++i) disk_free(tommy_array_get(&state->diskarr, i)); tommy_array_done(&state->diskarr); + tommy_list_foreach(&state->excludelist, (tommy_foreach_func*)filter_free); } void state_config(struct snapraid_state* state, const char* path, int verbose, int force) @@ -97,7 +100,7 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i s = strtoken(s); if (strcmp(tag, "block_size") == 0) { - ret = stru(s, &state->block_size); + ret = stru32(s, &state->block_size); if (ret != 0) { fprintf(stderr, "Invalid 'block_size' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); @@ -121,17 +124,24 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i fprintf(stderr, "Multiple 'parity' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } - snprintf(state->parity, sizeof(state->parity), "%s", s); + pathcpy(state->parity, sizeof(state->parity), s); } else if (strcmp(tag, "content") == 0) { if (*state->content) { fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } - snprintf(state->content, sizeof(state->content), "%s", s); + pathcpy(state->content, sizeof(state->content), s); } else if (strcmp(tag, "disk") == 0) { char* name = s; s = strtoken(s); tommy_array_insert(&state->diskarr, disk_alloc(name, s)); + } else if (strcmp(tag, "exclude") == 0) { + struct snapraid_filter* filter = filter_alloc(s); + if (!filter) { + fprintf(stderr, "Invalid 'exclude' specification '%s' in '%s' at line %u\n", s, path, line); + exit(EXIT_FAILURE); + } + tommy_list_insert_tail(&state->excludelist, &filter->node, filter); } else { fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path); exit(EXIT_FAILURE); @@ -164,7 +174,7 @@ void state_read(struct snapraid_state* state) count_file = 0; count_block = 0; - snprintf(path, sizeof(path), "%s", state->content); + pathcpy(path, sizeof(path), state->content); f = fopen(state->content, "r"); if (!f) { /* if not found, assume empty */ @@ -292,7 +302,7 @@ void state_read(struct snapraid_state* state) s = strtoken(s); hash = s; - ret = stru(pos, &v_pos); + ret = stru32(pos, &v_pos); if (ret != 0) { fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); @@ -367,7 +377,7 @@ void state_write(struct snapraid_state* state) printf("Saving state...\n"); - snprintf(path, sizeof(path), "%s.tmp", state->content); + pathprint(path, sizeof(path), "%s.tmp", state->content); f = fopen(path, "w"); if (!f) { fprintf(stderr, "Error opening for writing the state file '%s'\n", path); diff --git a/state.h b/state.h index f755cef7..7208ddd4 100644 --- a/state.h +++ b/state.h @@ -29,10 +29,11 @@ extern volatile int global_interrupt; struct snapraid_state { int verbose; /**< Verbose output. */ int force; /**< Forced dangerous operations. */ - unsigned block_size; /**< Block size in bytes. */ + uint32_t block_size; /**< Block size in bytes. */ char content[PATH_MAX]; /**< Path of the content file. */ char parity[PATH_MAX]; /**< Path of the parity file. */ tommy_array diskarr; /**< Disk array. */ + tommy_list excludelist; /**< List of exclusion. */ }; /** diff --git a/sync.c b/sync.c index 95091ba1..52266275 100644 --- a/sync.c +++ b/sync.c @@ -155,7 +155,7 @@ void state_sync(struct snapraid_state* state, block_off_t blockstart) exit(EXIT_FAILURE); } - snprintf(path, sizeof(path), "%s", state->parity); + pathcpy(path, sizeof(path), state->parity); f = parity_create(path, size); state_sync_process(state, f, blockstart, blockmax); diff --git a/test.conf b/test.conf index ea41a1c7..8c02358f 100644 --- a/test.conf +++ b/test.conf @@ -1,7 +1,7 @@ block_size 1 parity test/parity content test/content -disk d1 test/disk1 -disk d2 test/disk2 -disk d3 test/disk3 +disk d1 test/disk1/ +disk d2 test/disk2/ +disk d3 test/disk3/ diff --git a/tommylist.h b/tommylist.h index a90483ff..562e289d 100644 --- a/tommylist.h +++ b/tommylist.h @@ -305,5 +305,31 @@ tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list) return tommy_list_head(list) == 0; } +/** + * Calls the specified function for each element in the list. + */ +tommy_inline void tommy_list_foreach(tommy_list* list, tommy_foreach_func* func) +{ + tommy_node* node = tommy_list_head(list); + while (node) { + void* data = node->data; + node = node->next; + func(data); + } +} + +/** + * Calls the specified function with argument for each element in the list. + */ +tommy_inline void tommy_list_foreach_arg(tommy_list* list, tommy_foreach_arg_func* func, void* arg) +{ + tommy_node* node = tommy_list_head(list); + while (node) { + void* data = node->data; + node = node->next; + func(arg, data); + } +} + #endif diff --git a/tommytypes.h b/tommytypes.h index 79d6ef52..82f66165 100644 --- a/tommytypes.h +++ b/tommytypes.h @@ -203,6 +203,17 @@ typedef struct tommy_node_struct { */ typedef int tommy_compare_func(const void* void_a, const void* void_b); +/** + * Foreach function. + */ +typedef void tommy_foreach_func(void* void_a); + +/** + * Foreach function with an argument. + * The argument is always passed as first parameter. + */ +typedef void tommy_foreach_arg_func(void* void_arg, void* void_a); + /******************************************************************************/ /* bit hacks */ diff --git a/util.c b/util.c index e415fe2c..0c18fcbc 100644 --- a/util.c +++ b/util.c @@ -19,6 +19,198 @@ #include "util.h" +/****************************************************************************/ +/* string */ + +int strgets(char* s, unsigned size, FILE* f) +{ + char* i = s; + char* send = s + size; + int c; + + while (1) { +#if HAVE_FGETC_UNLOCKED + c = fgetc_unlocked(f); +#else + c = fgetc(f); +#endif + if (c == EOF || c == '\n') + break; + + *i++ = c; + + if (i == send) { + return -1; + } + } + + if (c == EOF) { + if ( +#if HAVE_FERROR_UNLOCKED + ferror_unlocked(f) +#else + ferror(f) +#endif + ) { + return -1; + } + if (i == s) + return 0; + } + + /* remove ending spaces */ + while (i != s && isspace(i[-1])) + --i; + *i = 0; + + return 1; +} + +int stru32(const char* s, uint32_t* value) +{ + uint32_t v; + + if (!*s) + return -1; + + v = 0; + while (*s>='0' && *s<='9') { + v *= 10; + v += *s - '0'; + ++s; + } + + if (*s) + return -1; + + *value = v; + + return 0; +} + +int stru64(const char* s, uint64_t* value) +{ + uint64_t v; + + if (!*s) + return -1; + + v = 0; + while (*s>='0' && *s<='9') { + v *= 10; + v += *s - '0'; + ++s; + } + + if (*s) + return -1; + + *value = v; + + return 0; +} + +static char strhexset[16] = "0123456789abcdef"; + +void strenchex(char* str, const void* void_data, unsigned data_len) +{ + const unsigned char* data = void_data; + unsigned i; + + for(i=0;i> 4]; + *str++ = strhexset[b & 0xF]; + } +} + +char* strdechex(void* void_data, unsigned data_len, char* str) +{ + unsigned char* data = void_data; + unsigned i; + + for(i=0;i= 'A' && c0 <= 'F') + b0 = c0 - 'A' + 10; + else if (c0 >= 'a' && c0 <= 'f') + b0 = c0 - 'a' + 10; + else if (c0 >= '0' && c0 <= '9') + b0 = c0 - '0'; + else + return str; + ++str; + + c1 = *str; + if (c1 >= 'A' && c1 <= 'F') + b1 = c1 - 'A' + 10; + else if (c1 >= 'a' && c1 <= 'f') + b1 = c1 - 'a' + 10; + else if (c1 >= '0' && c1 <= '9') + b1 = c1 - '0'; + else + return str; + ++str; + + data[i] = (b0 << 4) | b1; + } + + return 0; +} + +/****************************************************************************/ +/* path */ + +void pathcpy(char* str, size_t size, const char* src) +{ + size_t len = strlen(src); + + if (len + 1 >= size) { + fprintf(stderr, "Path too long\n"); + exit(EXIT_FAILURE); + } + + memcpy(str, src, len + 1); +} + +void pathprint(char* str, size_t size, const char* format, ...) +{ + int len; + va_list ap; + + va_start(ap, format); + len = vsnprintf(str, size, format, ap); + va_end(ap); + + if (len >= size) { + fprintf(stderr, "Path too long\n"); + exit(EXIT_FAILURE); + } +} + +void pathslash(char* str, size_t size) +{ + size_t len = strlen(str); + + if (len > 0 && str[len - 1] != '/') { + if (len + 2 >= size) { + fprintf(stderr, "Path too long\n"); + exit(EXIT_FAILURE); + } + + str[len] = '/'; + str[len+1] = 0; + } +} + +/****************************************************************************/ +/* mem */ + void* malloc_nofail(size_t size) { void* ptr = malloc(size); diff --git a/util.h b/util.h index 7ea362b7..2deb3533 100644 --- a/util.h +++ b/util.h @@ -21,6 +21,9 @@ /****************************************************************************/ /* string */ +/** + * Marks the end of the current token, and move to the next one. + */ static inline char* strtoken(char* s) { while (*s && *s != ' ') @@ -32,6 +35,9 @@ static inline char* strtoken(char* s) return s; } +/** + * Move to the next token skipping any separator. + */ static inline char* strskip(char* s) { while (*s == ' ') @@ -39,145 +45,54 @@ static inline char* strskip(char* s) return s; } -static inline int strgets(char* s, unsigned size, FILE* f) -{ - char* i = s; - char* send = s + size; - int c; +/** + * Gets a text line from a file. + * Returns -1 on error, 0 on oef, and 1 if more data is available. + */ +int strgets(char* s, unsigned size, FILE* f); - while (1) { -#if HAVE_FGETC_UNLOCKED - c = fgetc_unlocked(f); -#else - c = fgetc(f); -#endif +/** + * Convert a string to a 32 bit unsigned. + * Returns 0 on success. + */ +int stru32(const char* s, uint32_t* value); - if (c == EOF || c == '\n') - break; +/** + * Convert a string to a 64 bit unsigned. + * Returns 0 on success. + */ +int stru64(const char* s, uint64_t* value); - *i++ = c; +/** + * Encodes to hex. + */ +void strenchex(char* str, const void* void_data, unsigned data_len); - if (i == send) { - return -1; - } - } +/** + * Decodes from hex. + */ +char* strdechex(void* void_data, unsigned data_len, char* str); - if (c == EOF) { -#if HAVE_FERROR_UNLOCKED - if (ferror_unlocked(f)) { -#else - if (ferror(f)) { -#endif - return -1; - } - if (i == s) - return 0; - } +/****************************************************************************/ +/* path */ - /* remove ending spaces */ - while (i != s && isspace(i[-1])) - --i; - *i = 0; +/** + * Copies a path limiting the size. + * Aborts if too long. + */ +void pathcpy(char* str, size_t size, const char* src); - return 1; -} +/** + * Prints a path. + * Aborts if too long. + */ +void pathprint(char* str, size_t size, const char* format, ...); -static inline int stru(const char* s, unsigned* value) -{ - unsigned v; - - if (!*s) - return -1; - - v = 0; - while (*s>='0' && *s<='9') { - v *= 10; - v += *s - '0'; - ++s; - } - - if (*s) - return -1; - - *value = v; - - return 0; -} - -static inline int stru64(const char* s, uint64_t* value) -{ - uint64_t v; - - if (!*s) - return -1; - - v = 0; - while (*s>='0' && *s<='9') { - v *= 10; - v += *s - '0'; - ++s; - } - - if (*s) - return -1; - - *value = v; - - return 0; -} - -static char strhexset[16] = "0123456789abcdef"; - -static inline void strenchex(char* str, const void* void_data, unsigned data_len) -{ - const unsigned char* data = void_data; - unsigned i; - - for(i=0;i> 4]; - *str++ = strhexset[b & 0xF]; - } -} - -static inline char* strdechex(void* void_data, unsigned data_len, char* str) -{ - unsigned char* data = void_data; - unsigned i; - - for(i=0;i= 'A' && c0 <= 'F') - b0 = c0 - 'A' + 10; - else if (c0 >= 'a' && c0 <= 'f') - b0 = c0 - 'a' + 10; - else if (c0 >= '0' && c0 <= '9') - b0 = c0 - '0'; - else - return str; - ++str; - - c1 = *str; - if (c1 >= 'A' && c1 <= 'F') - b1 = c1 - 'A' + 10; - else if (c1 >= 'a' && c1 <= 'f') - b1 = c1 - 'a' + 10; - else if (c1 >= '0' && c1 <= '9') - b1 = c1 - '0'; - else - return str; - ++str; - - data[i] = (b0 << 4) | b1; - } - - return 0; -} +/** + * Ensures the presence of a terminating slash, if it isn't empty. + * Aborts if too long. + */ +void pathslash(char* str, size_t size); /****************************************************************************/ /* memory */