From 38e1f4e251cd95a1ae5608e432809d6a9dbc367c Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 22 Feb 2013 22:17:54 +0100 Subject: [PATCH] Adds the new pool command to create a virtual view for the array. --- HISTORY | 4 + Makefile.am | 14 ++- configure.ac | 5 +- pool.c | 217 ++++++++++++++++++++++++++++++++++ snapraid.1 | 71 +++++++++-- snapraid.c | 7 ++ snapraid.conf.example | 8 ++ snapraid.conf.example.windows | 8 ++ snapraid.d | 65 ++++++++-- snapraid.txt | 90 ++++++++++---- state.c | 67 ++++++++++- state.h | 9 +- test-raid5.conf | 1 + 13 files changed, 514 insertions(+), 52 deletions(-) create mode 100644 pool.c diff --git a/HISTORY b/HISTORY index 718fcacd..a21fcb15 100644 --- a/HISTORY +++ b/HISTORY @@ -3,7 +3,11 @@ SnapRAID HISTORY 2.2 2013/2 ========== + * Added pooling support with the new 'pool' command. It creates a virtual view + of the array using symbolic links pointing at the original files. * Added support also for hardlinks and empty directories. + * Added symlinks support to Windows. Note that Windows junctions are still + not supported. 2.1 2013/1 ========== diff --git a/Makefile.am b/Makefile.am index 878b473d..416d71ff 100644 --- a/Makefile.am +++ b/Makefile.am @@ -12,6 +12,7 @@ snapraid_SOURCES = \ check.c \ dry.c \ dup.c \ + pool.c \ parity.c \ handle.c \ fnmatch.c \ @@ -94,6 +95,9 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf dup $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync +if HAVE_SYMLINK + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool +endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -a check echo --- Create some files, sync raid6 and check @@ -101,6 +105,9 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync +if HAVE_SYMLINK + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool +endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf -a check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check @@ -125,6 +132,9 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf dup $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check +if HAVE_SYMLINK + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool +endif echo --- Extend raid5 to raid6 with fix and check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid6.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log @@ -268,13 +278,13 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check - echo --- Delete some dirs, fix and check with raid5 + echo --- Delete some dirs in a disk, fix and check with raid5 rm -r test/disk2/_a rm -r test/disk2/_b $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check - echo --- Delete some dirs, fix and check with raid6 using the -d option for each disk + echo --- Delete some dirs in two disks, fix and check with raid6 using the -d option for each disk rm -r test/disk2/_a rm -r test/disk2/_b rm -r test/disk3/_a diff --git a/configure.ac b/configure.ac index 8a4db53d..135eec5d 100644 --- a/configure.ac +++ b/configure.ac @@ -60,9 +60,10 @@ AC_CHECK_CC_OPT([-Wextra], CFLAGS="$CFLAGS -Wextra", []) dnl Checks for test environment AS_CASE([$host], - [*-*-mingw*], [TESTENV="$WINE"], - [] + [*-*-mingw*], [TESTENV="$WINE"], + [SYMLINK=1] ) +AM_CONDITIONAL(HAVE_SYMLINK, [test x"$SYMLINK" != x]) AC_ARG_ENABLE([profiler], [AS_HELP_STRING([--enable-profiler],[enable the use of gprof for code coverage])], diff --git a/pool.c b/pool.c new file mode 100644 index 00000000..36aea366 --- /dev/null +++ b/pool.c @@ -0,0 +1,217 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "util.h" +#include "elem.h" +#include "state.h" + +/** + * Clean a directory tree removing all the symlinks and empty directories. + * Return == 0 if the directory is empty, and it can be removed + */ +static int clean_dir(struct snapraid_state* state, const char* dir) +{ + DIR* d; + int ignored = 0; + + d = opendir(dir); + if (!d) { + fprintf(stderr, "Error opening pool directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + } + + while (1) { + char path_next[PATH_MAX]; + struct stat st; + const char* name; + struct dirent* dd; + + /* clear errno to detect erroneous conditions */ + errno = 0; + dd = readdir(d); + if (dd == 0 && errno != 0) { + fprintf(stderr, "Error reading pool directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + } + if (dd == 0 && errno == 0) { + break; /* finished */ + } + + /* skip "." and ".." files */ + name = dd->d_name; + if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0))) + continue; + + pathprint(path_next, sizeof(path_next), "%s%s", dir, name); + +#if HAVE_DIRENT_LSTAT + /* convert dirent to lstat result */ + dirent_lstat(dd, &st); +#else + /* get lstat info about the file */ + if (lstat(path_next, &st) != 0) { + fprintf(stderr, "Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno)); + exit(EXIT_FAILURE); + } +#endif + + if (S_ISLNK(st.st_mode)) { + int ret; + + /* delete the link */ + ret = remove(path_next); + if (ret < 0) { + fprintf(stderr, "Error removing symlink '%s'. %s.\n", path_next, strerror(errno)); + exit(EXIT_FAILURE); + } + } else if (S_ISDIR(st.st_mode)) { + /* recurse */ + pathslash(path_next, sizeof(path_next)); + if (clean_dir(state, path_next) == 0) { + int ret; + + /* directory is empty, try to remove it */ + ret = rmdir(path_next); + if (ret < 0) { +#ifdef _WIN32 + if (errno == EACCES) { + /* in Windows just ignore EACCES errors removing directories */ + /* because it could happen that the directory is in use */ + /* and it cannot be removed */ + fprintf(stderr, "Directory '%s' not removed because it's in use. \n", path_next); + ignored = 1; + } else +#endif + { + fprintf(stderr, "Error removing pool directory '%s'. %s.\n", path_next, strerror(errno)); + exit(EXIT_FAILURE); + } + } + } else { + /* something was ignored inside the subdir */ + ignored = 1; + } + } else { + ignored = 1; + if (state->verbose) { + printf("Ignoring pool file '%s'\n", path_next); + } + } + } + + if (closedir(d) != 0) { + fprintf(stderr, "Error closing pool directory '%s'. %s.\n", dir, strerror(errno)); + exit(EXIT_FAILURE); + } + + return ignored; +} + +/** + * Creates a link to the specified disk entry. + */ +static void make_link(const char* pool_dir, struct snapraid_disk* disk, const char* sub) +{ + char path[PATH_MAX]; + char linkto[PATH_MAX]; + int ret; + + /* make the paths */ + pathprint(path, sizeof(path), "%s%s", pool_dir, sub); + pathprint(linkto, sizeof(linkto), "%s%s", disk->dir, sub); + + /* create the ancestor directories */ + ret = mkancestor(path); + if (ret != 0) { + exit(EXIT_FAILURE); + } + + /* create the symlink */ + ret = symlink(linkto, path); + if (ret != 0) { + if (errno == EEXIST) { + fprintf(stderr, "warning: '%s' already exists\n", path); +#ifdef _WIN32 + } else if (errno == EPERM) { + fprintf(stderr, "You must run as Adminstrator to be able to create symlinks.\n"); + exit(EXIT_FAILURE); +#endif + } else { + fprintf(stderr, "Error writing symlink '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + } +} + +void state_pool(struct snapraid_state* state) +{ + tommy_node* i; + char pool_dir[PATH_MAX]; + int ret; + unsigned count; + + if (state->pool[0] == 0) { + fprintf(stderr, "To use the 'pool' command you must set the pool directory in the configuration file\n"); + exit(EXIT_FAILURE); + } + + /* pool directory with final slash */ + pathprint(pool_dir, sizeof(pool_dir), "%s", state->pool); + pathslash(pool_dir, sizeof(pool_dir)); + + /* create it if missing */ + ret = mkancestor(pool_dir); + if (ret != 0) { + exit(EXIT_FAILURE); + } + + /* first clear the previous pool tree */ + clean_dir(state, pool_dir); + + /* for each disk */ + count = 0; + for(i=state->disklist;i!=0;i=i->next) { + tommy_node* j; + struct snapraid_disk* disk = i->data; + + /* for each file */ + for(j=disk->filelist;j!=0;j=j->next) { + struct snapraid_file* file = j->data; + make_link(pool_dir, disk, file->sub); + ++count; + } + + /* for each link */ + for(j=disk->linklist;j!=0;j=j->next) { + struct snapraid_link* link = j->data; + make_link(pool_dir, disk, link->sub); + ++count; + } + + /* we ignore empty dirs in disk->dir */ + } + + if (!state->gui) { + if (count) + printf("%u links created\n", count); + else + printf("No link created\n"); + } +} + diff --git a/snapraid.1 b/snapraid.1 index 65d4391e..ae67a457 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -30,7 +30,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] .PD 0 .PP .PD - sync|diff|dup|check|fix + sync|pool|diff|dup|check|fix .PD 0 .PP .PD @@ -114,7 +114,7 @@ With a Backup you are able to recover from a complete failure of the whole disk array. .IP \(bu Only file, timestamps, symlinks and hardlinks are saved. -Permissions, ownership, extended attributes are not saved. +Permissions, ownership and extended attributes are not saved. .PD .SH GETTING STARTED To use SnapRAID you need to first select one disk of your disk array @@ -250,7 +250,37 @@ When this command completes, your data is SAFE. .PP At this point you can start using your array as you like, and periodically update the redundancy information running the \(a"sync\(a" command. +.SS Pooling +To have all the files in your array shown in the same directory tree, +you can enable \(a"pooling\(a", that consists in creating a virtual view of all +the files in your array using symbolic links. +You can configure the \(a"pooling\(a" directory in the configuration file with: .PP +.RS 4 +pool /pool +.PD 0 +.PP +.PD +.RE +.PP +or, if you are in Windows, with: +.PP +.RS 4 +pool C:\(rspool +.PD 0 +.PP +.PD +.RE +.PP +and then run the \(a"pool\(a" command. +.PP +.RS 4 +snapraid pool +.PD 0 +.PP +.PD +.RE +.SS Checking & Fixing To check the integrity of your data you can use the \(a"check\(a" command: .PP .RS 4 @@ -275,7 +305,7 @@ Note that the fix command will revert your data at the state of the last \(a"sync\(a" command executed. It works like a snapshot was taken in \(a"sync\(a". .PP -In this regard snapraid is more like a backup program than a RAID +In this regard SnapRAID is more like a backup program than a RAID system. For example, you can use it to recover from an accidentally deleted directory, simply running the fix command like. .PP @@ -322,21 +352,33 @@ without losing the work already done. .PP The \(a"content\(a", \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary. The files in the array are NOT modified. +.SS pool +Creates or updates in the \(a"pooling\(a" directory a virtual view of all +the files of your disk array. +.PP +The files are not really copied here, but just linked using +symbolic links. +.PP +When updating, all the present symbolic links and empty +subdirectories are deleted and replaced with the new +view of the array. Any othe regular file is left in place. +.PP +Nothing is modified outside the pool directory. .SS diff Lists all the files modified from the last \(a"sync\(a" command that have to recompute their redundancy data. .PP Nothing is modified. .SS dup -Lists all the duplicate files. Two files are assumed equal if their hashes -are matching. The effective data is not read. +Lists all the duplicate files. Two files are assumed equal if their +hashes are matching. The effective data is not read. .PP Nothing is modified. .SS check Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved in the previous \(a"sync\(a" command. -If you use the \(hyA, \(hy\(hyaudit\(hyonly option, only the file +If you use the \(hya, \(hy\(hyaudit\(hyonly option, only the file data is checked, and the redundandy data is ignored. .PP Files are identified by path, and checked by content. @@ -381,6 +423,8 @@ commands. Only the files present in the specified disk are processed. You must specify a disk name as named in the configuration file. +In \(a"check\(a", you can make it faster, specifing also \(hya, \(hy\(hyaudit\(hyonly +option, to avoid to access other disks to check parity data. If you combine \(hy\(hyfilter and \(hy\(hyfilter\(hydisk options, only files matching both the two groups of rules are selected. This option can be used many times. @@ -405,9 +449,6 @@ size and time with second precision is assumed identical at the previous one, and not resynched. The nanosecond part of timestamps is ignored because most of copy programs are not able to correctly restore it. -Note that if you use this option, all the hard\(hylinks will be -stored independently wasting space. So, if you use hard\(hylinks, -it\(cqs better to don\(cqt use it. This option can be used only with the \(a"sync\(a" and \(a"diff\(a" commands. .TP .B \(hyZ, \(hy\(hyforce\(hyzero @@ -503,7 +544,7 @@ keep the NAME fixed. You should use one option for each disk of the array. .SS nohidden Excludes all the hidden files and directory. -In Unix hidden files are the ones starting with \(cq.\(cq. +In Unix hidden files are the ones starting with \(a".\(a". In Windows they are the ones with the hidden attribute. .SS exclude/include PATTERN Defines the file or directory patterns to exclude and include @@ -554,11 +595,14 @@ going to waste 1.2 GiB. .SS autosave SIZE_IN_GIBIBYTES Automatically save the state when synching after the specied amount of GiB processed. -This option is useful to avoid to restart from scratch long \(cqsync\(cq +This option is useful to avoid to restart from scratch long \(a"sync\(a" commands interrupted by a machine crash, or any other event that may interrupt SnapRAID. The SIZE argument is specified in gibibytes. Where one gibi bytes is 1073741824 bytes. +.SS pool DIR +Defines the pooling directory where the virtual view of the disk +array is created using the \(a"pool\(a" command. .SS Examples An example of a typical configuration for Unix is: .PP @@ -865,7 +909,7 @@ As paranoid check, you can now run a whole \(a"check\(a" command to ensure that everything is OK. .PP .RS 4 -snapraid \(hyd NAME check +snapraid \(hyd NAME \(hya check .PD 0 .PP .PD @@ -873,6 +917,9 @@ snapraid \(hyd NAME check .PP Where NAME is the name of the disk, like \(a"d1\(a" as in our previous example. .PP +The options \(hyd and \(hya tell SnapRAID to check only the specified disk, +and ignore all the other data and redundancy disks. +.PP This command will take a long time. .SS STEP 4 \(hy> Sync Run the \(a"sync\(a" command to resyncronize the array with the new disk. diff --git a/snapraid.c b/snapraid.c index 72ac9836..3ccc9223 100644 --- a/snapraid.c +++ b/snapraid.c @@ -122,6 +122,7 @@ void signal_handler(int signal) #define OPERATION_FIX 3 #define OPERATION_DRY 4 #define OPERATION_DUP 5 +#define OPERATION_POOL 6 int main(int argc, char* argv[]) { @@ -276,6 +277,8 @@ int main(int argc, char* argv[]) operation = OPERATION_DRY; } else if (strcmp(argv[optind], "dup") == 0) { operation = OPERATION_DUP; + } else if (strcmp(argv[optind], "pool") == 0) { + operation = OPERATION_POOL; } else { fprintf(stderr, "Unknown command '%s'\n", argv[optind]); exit(EXIT_FAILURE); @@ -386,6 +389,10 @@ int main(int argc, char* argv[]) state_read(&state); state_dup(&state); + } else if (operation == OPERATION_POOL) { + state_read(&state); + + state_pool(&state); } else { state_read(&state); diff --git a/snapraid.conf.example b/snapraid.conf.example index 9148d1f9..266025ec 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -56,3 +56,11 @@ block_size 256 # Format: "autosave_size SIZE_IN_GiB" # autosize 500 +# Defines the pooling directory where the virtual view of the disk +# array is created using the "pool" command. +# The files are not really copied here, but just linked using +# symbolic links. +# This directory must be outside the array. +# Format: "pool DIR" +#pool /pool + diff --git a/snapraid.conf.example.windows b/snapraid.conf.example.windows index 0fb51ea2..598901a4 100644 --- a/snapraid.conf.example.windows +++ b/snapraid.conf.example.windows @@ -57,3 +57,11 @@ block_size 256 # Format: "autosave_size SIZE_IN_GiB" # autosize 500 +# Defines the pooling directory where the virtual view of the disk +# array is created using the "pool" command. +# The files are not really copied here, but just linked using +# symbolic links. +# This directory must be outside the array. +# Format: "pool DIR" +#pool C:\pool + diff --git a/snapraid.d b/snapraid.d index fa835e6d..3ef8f73e 100644 --- a/snapraid.d +++ b/snapraid.d @@ -9,7 +9,7 @@ Synopsis : [-Z, --force-zero] [-E, --force-empty] : [-s, --start BLKSTART] [-t, --count BLKCOUNT] : [-v, --verbose] - : sync|diff|dup|check|fix + : sync|pool|diff|dup|check|fix :snapraid [-V, --version] [-h, --help] @@ -71,7 +71,7 @@ Limitations With a Backup you are able to recover from a complete failure of the whole disk array. * Only file, timestamps, symlinks and hardlinks are saved. - Permissions, ownership, extended attributes are not saved. + Permissions, ownership and extended attributes are not saved. Getting Started To use SnapRAID you need to first select one disk of your disk array @@ -143,6 +143,23 @@ Getting Started At this point you can start using your array as you like, and periodically update the redundancy information running the "sync" command. + Pooling + To have all the files in your array shown in the same directory tree, + you can enable "pooling", that consists in creating a virtual view of all + the files in your array using symbolic links. + You can configure the "pooling" directory in the configuration file with: + + :pool /pool + + or, if you are in Windows, with: + + :pool C:\pool + + and then run the "pool" command. + + :snapraid pool + + Checking & Fixing To check the integrity of your data you can use the "check" command: :snapraid check @@ -157,7 +174,7 @@ Getting Started last "sync" command executed. It works like a snapshot was taken in "sync". - In this regard snapraid is more like a backup program than a RAID + In this regard SnapRAID is more like a backup program than a RAID system. For example, you can use it to recover from an accidentally deleted directory, simply running the fix command like. @@ -192,6 +209,19 @@ Commands The "content", "parity" and "q-parity" files are modified if necessary. The files in the array are NOT modified. + pool + Creates or updates in the "pooling" directory a virtual view of all + the files of your disk array. + + The files are not really copied here, but just linked using + symbolic links. + + When updating, all the present symbolic links and empty + subdirectories are deleted and replaced with the new + view of the array. Any othe regular file is left in place. + + Nothing is modified outside the pool directory. + diff Lists all the files modified from the last "sync" command that have to recompute their redundancy data. @@ -199,8 +229,8 @@ Commands Nothing is modified. dup - Lists all the duplicate files. Two files are assumed equal if their hashes - are matching. The effective data is not read. + Lists all the duplicate files. Two files are assumed equal if their + hashes are matching. The effective data is not read. Nothing is modified. @@ -208,7 +238,7 @@ Commands Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved in the previous "sync" command. - If you use the -A, --audit-only option, only the file + If you use the -a, --audit-only option, only the file data is checked, and the redundandy data is ignored. Files are identified by path, and checked by content. @@ -255,6 +285,8 @@ Options Only the files present in the specified disk are processed. You must specify a disk name as named in the configuration file. + In "check", you can make it faster, specifing also -a, --audit-only + option, to avoid to access other disks to check parity data. If you combine --filter and --filter-disk options, only files matching both the two groups of rules are selected. This option can be used many times. @@ -380,7 +412,7 @@ Configuration nohidden Excludes all the hidden files and directory. - In Unix hidden files are the ones starting with '.'. + In Unix hidden files are the ones starting with ".". In Windows they are the ones with the hidden attribute. exclude/include PATTERN @@ -431,12 +463,16 @@ Configuration autosave SIZE_IN_GIBIBYTES Automatically save the state when synching after the specied amount of GiB processed. - This option is useful to avoid to restart from scratch long 'sync' + This option is useful to avoid to restart from scratch long "sync" commands interrupted by a machine crash, or any other event that may interrupt SnapRAID. The SIZE argument is specified in gibibytes. Where one gibi bytes is 1073741824 bytes. + pool DIR + Defines the pooling directory where the virtual view of the disk + array is created using the "pool" command. + Examples An example of a typical configuration for Unix is: @@ -554,7 +590,7 @@ Recovering Then proceed with the following steps. - STEP 1 -> Reconfigure + STEP 1 -> Reconfigure You need some space to recover, even better if you already have an additional disk, but in case, also an external USB or remote one is enough. @@ -570,7 +606,7 @@ Recovering :disk d1 /mnt/new_spare_disk/ - STEP 2 -> Fix + STEP 2 -> Fix Run the fix command, storing the log in an external file with: :snapraid -d NAME fix 2>fix.log @@ -596,17 +632,20 @@ Recovering but take care that after synching you will no more able to retry the "fix" command! - STEP 3 -> Check + STEP 3 -> Check As paranoid check, you can now run a whole "check" command to ensure that everything is OK. - :snapraid -d NAME check + :snapraid -d NAME -a check Where NAME is the name of the disk, like "d1" as in our previous example. + The options -d and -a tell SnapRAID to check only the specified disk, + and ignore all the other data and redundancy disks. + This command will take a long time. - STEP 4 -> Sync + STEP 4 -> Sync Run the "sync" command to resyncronize the array with the new disk. To avoid a long time sync you can use the "--find-by-name" option to diff --git a/snapraid.txt b/snapraid.txt index 17254b41..9b29a622 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -13,7 +13,7 @@ snapraid [-c, --conf CONFIG] [-Z, --force-zero] [-E, --force-empty] [-s, --start BLKSTART] [-t, --count BLKCOUNT] [-v, --verbose] - sync|diff|dup|check|fix + sync|pool|diff|dup|check|fix snapraid [-V, --version] [-h, --help] @@ -81,7 +81,7 @@ Other limitations are: With a Backup you are able to recover from a complete failure of the whole disk array. * Only file, timestamps, symlinks and hardlinks are saved. - Permissions, ownership, extended attributes are not saved. + Permissions, ownership and extended attributes are not saved. 4 GETTING STARTED @@ -156,6 +156,27 @@ When this command completes, your data is SAFE. At this point you can start using your array as you like, and periodically update the redundancy information running the "sync" command. +4.1 Pooling +----------- + +To have all the files in your array shown in the same directory tree, +you can enable "pooling", that consists in creating a virtual view of all +the files in your array using symbolic links. +You can configure the "pooling" directory in the configuration file with: + + pool /pool + +or, if you are in Windows, with: + + pool C:\pool + +and then run the "pool" command. + + snapraid pool + +4.2 Checking & Fixing +--------------------- + To check the integrity of your data you can use the "check" command: snapraid check @@ -170,7 +191,7 @@ Note that the fix command will revert your data at the state of the last "sync" command executed. It works like a snapshot was taken in "sync". -In this regard snapraid is more like a backup program than a RAID +In this regard SnapRAID is more like a backup program than a RAID system. For example, you can use it to recover from an accidentally deleted directory, simply running the fix command like. @@ -210,7 +231,22 @@ without losing the work already done. The "content", "parity" and "q-parity" files are modified if necessary. The files in the array are NOT modified. -5.2 diff +5.2 pool +-------- + +Creates or updates in the "pooling" directory a virtual view of all +the files of your disk array. + +The files are not really copied here, but just linked using +symbolic links. + +When updating, all the present symbolic links and empty +subdirectories are deleted and replaced with the new +view of the array. Any othe regular file is left in place. + +Nothing is modified outside the pool directory. + +5.3 diff -------- Lists all the files modified from the last "sync" command that @@ -218,27 +254,27 @@ have to recompute their redundancy data. Nothing is modified. -5.3 dup +5.4 dup ------- -Lists all the duplicate files. Two files are assumed equal if their hashes -are matching. The effective data is not read. +Lists all the duplicate files. Two files are assumed equal if their +hashes are matching. The effective data is not read. Nothing is modified. -5.4 check +5.5 check --------- Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved in the previous "sync" command. -If you use the -A, --audit-only option, only the file +If you use the -a, --audit-only option, only the file data is checked, and the redundandy data is ignored. Files are identified by path, and checked by content. Nothing is modified. -5.5 fix +5.6 fix ------- Checks and fix all the files. It's like "check" but it @@ -284,6 +320,8 @@ SnapRAID provides the following options: Only the files present in the specified disk are processed. You must specify a disk name as named in the configuration file. + In "check", you can make it faster, specifing also -a, --audit-only + option, to avoid to access other disks to check parity data. If you combine --filter and --filter-disk options, only files matching both the two groups of rules are selected. This option can be used many times. @@ -308,9 +346,6 @@ SnapRAID provides the following options: the previous one, and not resynched. The nanosecond part of timestamps is ignored because most of copy programs are not able to correctly restore it. - Note that if you use this option, all the hard-links will be - stored independently wasting space. So, if you use hard-links, - it's better to don't use it. This option can be used only with the "sync" and "diff" commands. -Z, --force-zero @@ -425,7 +460,7 @@ You should use one option for each disk of the array. ------------ Excludes all the hidden files and directory. -In Unix hidden files are the ones starting with '.'. +In Unix hidden files are the ones starting with ".". In Windows they are the ones with the hidden attribute. 7.6 exclude/include PATTERN @@ -480,12 +515,16 @@ going to waste 1.2 GiB. ---- 7.7.1 autosave SIZE_IN_GIBIBYTES ---- Automatically save the state when synching after the specied amount of GiB processed. -This option is useful to avoid to restart from scratch long 'sync' +This option is useful to avoid to restart from scratch long "sync" commands interrupted by a machine crash, or any other event that may interrupt SnapRAID. The SIZE argument is specified in gibibytes. Where one gibi bytes is 1073741824 bytes. +---- 7.7.2 pool DIR ---- +Defines the pooling directory where the virtual view of the disk +array is created using the "pool" command. + 7.8 Examples ------------ @@ -611,7 +650,9 @@ scheduled SnapRAID nightly sync. Then proceed with the following steps. ----- 9.0.1 STEP 1 -> Reconfigure ---- +9.1 STEP 1 -> Reconfigure +------------------------- + You need some space to recover, even better if you already have an additional disk, but in case, also an external USB or remote one is enough. @@ -627,7 +668,9 @@ to: disk d1 /mnt/new_spare_disk/ ----- 9.0.2 STEP 2 -> Fix ---- +9.2 STEP 2 -> Fix +----------------- + Run the fix command, storing the log in an external file with: snapraid -d NAME fix 2>fix.log @@ -653,17 +696,24 @@ If you are satisfied of the recovering, you can now proceed further, but take care that after synching you will no more able to retry the "fix" command! ----- 9.0.3 STEP 3 -> Check ---- +9.3 STEP 3 -> Check +------------------- + As paranoid check, you can now run a whole "check" command to ensure that everything is OK. - snapraid -d NAME check + snapraid -d NAME -a check Where NAME is the name of the disk, like "d1" as in our previous example. +The options -d and -a tell SnapRAID to check only the specified disk, +and ignore all the other data and redundancy disks. + This command will take a long time. ----- 9.0.4 STEP 4 -> Sync ---- +9.4 STEP 4 -> Sync +------------------ + Run the "sync" command to resyncronize the array with the new disk. To avoid a long time sync you can use the "--find-by-name" option to diff --git a/state.c b/state.c index b8e5cb22..308ea490 100644 --- a/state.c +++ b/state.c @@ -38,6 +38,8 @@ void state_init(struct snapraid_state* state) state->parity_device = 0; state->qarity[0] = 0; state->qarity_device = 0; + state->pool[0] = 0; + state->pool_device = 0; state->level = 1; /* default is the lowest protection */ state->hash = HASH_MURMUR3; /* default is the fastest */ tommy_list_init(&state->disklist); @@ -123,6 +125,11 @@ static void state_config_check(struct snapraid_state* state, const char* path, i exit(EXIT_FAILURE); } + if (state->pool[0] != 0 && disk->device == state->pool_device) { + fprintf(stderr, "Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool); + exit(EXIT_FAILURE); + } + ++diskcount; } @@ -148,12 +155,30 @@ static void state_config_check(struct snapraid_state* state, const char* path, i fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n"); exit(EXIT_FAILURE); } + + if (state->pool[0] != 0 && state->pool_device == 0) { + fprintf(stderr, "Disk '%s' has a zero serial number.\n", state->pool); + fprintf(stderr, "This is not necessarely wrong, but for using SnapRAID\n"); + fprintf(stderr, "it's better to change the serial number of the disk.\n"); + fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n"); + exit(EXIT_FAILURE); + } #endif if (state->qarity[0] != 0 && state->parity_device == state->qarity_device) { fprintf(stderr, "Parity '%s' and '%s' are on the same device.\n", state->parity, state->qarity); exit(EXIT_FAILURE); } + + if (state->pool[0] != 0 && state->pool_device == state->parity_device) { + fprintf(stderr, "Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity); + exit(EXIT_FAILURE); + } + + if (state->pool[0] != 0 && state->qarity[0] != 0 && state->pool_device == state->qarity_device) { + fprintf(stderr, "Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->qarity); + exit(EXIT_FAILURE); + } } /* count the content files */ @@ -283,7 +308,7 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i pathimport(state->parity, sizeof(state->parity), buffer); - /* get the device of the directory containing the content file */ + /* get the device of the directory containing the parity file */ pathimport(device, sizeof(device), buffer); slash = strrchr(device, '/'); if (slash) @@ -319,7 +344,7 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i pathimport(state->qarity, sizeof(state->qarity), buffer); - /* get the device of the directory containing the content file */ + /* get the device of the directory containing the qparity file */ pathimport(device, sizeof(device), buffer); slash = strrchr(device, '/'); if (slash) @@ -335,6 +360,42 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i /* we have two level of parity */ state->level = 2; + } else if (strcmp(tag, "pool") == 0) { + char device[PATH_MAX]; + char* slash; + struct stat st; + + if (*state->pool) { + fprintf(stderr, "Multiple 'pool' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + ret = sgetlasttok(f, buffer, sizeof(buffer)); + if (ret < 0) { + fprintf(stderr, "Invalid 'pool' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + if (!*buffer) { + fprintf(stderr, "Empty 'pool' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + + pathimport(state->pool, sizeof(state->pool), buffer); + + /* get the device of the directory containing the pool tree */ + pathimport(device, sizeof(device), buffer); + slash = strrchr(device, '/'); + if (slash) + *slash = 0; + else + pathcpy(device, sizeof(device), "."); + if (stat(device, &st) != 0) { + fprintf(stderr, "Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", device, path, line); + exit(EXIT_FAILURE); + } + + state->pool_device = st.st_dev; } else if (strcmp(tag, "content") == 0) { struct snapraid_content* content; char device[PATH_MAX]; @@ -528,6 +589,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i fprintf(stderr, "parity:%s\n", state->parity); if (state->qarity[0] != 0) fprintf(stderr, "qarity:%s\n", state->qarity); + if (state->pool[0] != 0) + fprintf(stderr, "pool:%s\n", state->pool); fflush(stderr); } } diff --git a/state.h b/state.h index d2ccae68..42a37653 100644 --- a/state.h +++ b/state.h @@ -42,6 +42,8 @@ struct snapraid_state { uint64_t parity_device; /**< Device identifier of the parity. */ char qarity[PATH_MAX]; /**< Path of the qarity file. */ uint64_t qarity_device; /**< Device identifier of the qarity. */ + char pool[PATH_MAX]; /**< Path of the pool tree. */ + uint64_t pool_device; /**< Device identifier of the pool. */ unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */ unsigned hash; /**< Hash kind used. */ tommy_list contentlist; /**< List of content files. */ @@ -103,10 +105,15 @@ void state_check(struct snapraid_state* state, int check, int fix, block_off_t b void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount); /** - * Find duplicates. + * Finds duplicates. */ void state_dup(struct snapraid_state* state); +/** + * Creates pool tree. + */ +void state_pool(struct snapraid_state* state); + /** * Filter files, symlinks and dirs. * Apply an additional filter to the list currently loaded. diff --git a/test-raid5.conf b/test-raid5.conf index 0fdb8f46..5e2ef476 100644 --- a/test-raid5.conf +++ b/test-raid5.conf @@ -6,3 +6,4 @@ disk disk1 test/disk1/ disk disk2 test/disk2/ disk disk3 test/disk3/ exclude *.unrecoverable +pool test/pool