Adds the new pool command to create a virtual view for the array.

This commit is contained in:
Andrea Mazzoleni 2013-02-22 22:17:54 +01:00
parent dc1909ea03
commit 38e1f4e251
13 changed files with 514 additions and 52 deletions

View file

@ -3,7 +3,11 @@ SnapRAID HISTORY
2.2 2013/2
==========
* Added pooling support with the new 'pool' command. It creates a virtual view
of the array using symbolic links pointing at the original files.
* Added support also for hardlinks and empty directories.
* Added symlinks support to Windows. Note that Windows junctions are still
not supported.
2.1 2013/1
==========

View file

@ -12,6 +12,7 @@ snapraid_SOURCES = \
check.c \
dry.c \
dup.c \
pool.c \
parity.c \
handle.c \
fnmatch.c \
@ -94,6 +95,9 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync
if HAVE_SYMLINK
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -a check
echo --- Create some files, sync raid6 and check
@ -101,6 +105,9 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
if HAVE_SYMLINK
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf -a check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
@ -125,6 +132,9 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
if HAVE_SYMLINK
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
endif
echo --- Extend raid5 to raid6 with fix and check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
@ -268,13 +278,13 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete some dirs, fix and check with raid5
echo --- Delete some dirs in a disk, fix and check with raid5
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete some dirs, fix and check with raid6 using the -d option for each disk
echo --- Delete some dirs in two disks, fix and check with raid6 using the -d option for each disk
rm -r test/disk2/_a
rm -r test/disk2/_b
rm -r test/disk3/_a

View file

@ -60,9 +60,10 @@ AC_CHECK_CC_OPT([-Wextra], CFLAGS="$CFLAGS -Wextra", [])
dnl Checks for test environment
AS_CASE([$host],
[*-*-mingw*], [TESTENV="$WINE"],
[]
[*-*-mingw*], [TESTENV="$WINE"],
[SYMLINK=1]
)
AM_CONDITIONAL(HAVE_SYMLINK, [test x"$SYMLINK" != x])
AC_ARG_ENABLE([profiler],
[AS_HELP_STRING([--enable-profiler],[enable the use of gprof for code coverage])],

217
pool.c Normal file
View file

@ -0,0 +1,217 @@
/*
* Copyright (C) 2013 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "portable.h"
#include "util.h"
#include "elem.h"
#include "state.h"
/**
* Clean a directory tree removing all the symlinks and empty directories.
* Return == 0 if the directory is empty, and it can be removed
*/
static int clean_dir(struct snapraid_state* state, const char* dir)
{
DIR* d;
int ignored = 0;
d = opendir(dir);
if (!d) {
fprintf(stderr, "Error opening pool directory '%s'. %s.\n", dir, strerror(errno));
exit(EXIT_FAILURE);
}
while (1) {
char path_next[PATH_MAX];
struct stat st;
const char* name;
struct dirent* dd;
/* clear errno to detect erroneous conditions */
errno = 0;
dd = readdir(d);
if (dd == 0 && errno != 0) {
fprintf(stderr, "Error reading pool directory '%s'. %s.\n", dir, strerror(errno));
exit(EXIT_FAILURE);
}
if (dd == 0 && errno == 0) {
break; /* finished */
}
/* skip "." and ".." files */
name = dd->d_name;
if (name[0] == '.' && (name[1] == 0 || (name[1] == '.' && name[2] == 0)))
continue;
pathprint(path_next, sizeof(path_next), "%s%s", dir, name);
#if HAVE_DIRENT_LSTAT
/* convert dirent to lstat result */
dirent_lstat(dd, &st);
#else
/* get lstat info about the file */
if (lstat(path_next, &st) != 0) {
fprintf(stderr, "Error in stat file/directory '%s'. %s.\n", path_next, strerror(errno));
exit(EXIT_FAILURE);
}
#endif
if (S_ISLNK(st.st_mode)) {
int ret;
/* delete the link */
ret = remove(path_next);
if (ret < 0) {
fprintf(stderr, "Error removing symlink '%s'. %s.\n", path_next, strerror(errno));
exit(EXIT_FAILURE);
}
} else if (S_ISDIR(st.st_mode)) {
/* recurse */
pathslash(path_next, sizeof(path_next));
if (clean_dir(state, path_next) == 0) {
int ret;
/* directory is empty, try to remove it */
ret = rmdir(path_next);
if (ret < 0) {
#ifdef _WIN32
if (errno == EACCES) {
/* in Windows just ignore EACCES errors removing directories */
/* because it could happen that the directory is in use */
/* and it cannot be removed */
fprintf(stderr, "Directory '%s' not removed because it's in use. \n", path_next);
ignored = 1;
} else
#endif
{
fprintf(stderr, "Error removing pool directory '%s'. %s.\n", path_next, strerror(errno));
exit(EXIT_FAILURE);
}
}
} else {
/* something was ignored inside the subdir */
ignored = 1;
}
} else {
ignored = 1;
if (state->verbose) {
printf("Ignoring pool file '%s'\n", path_next);
}
}
}
if (closedir(d) != 0) {
fprintf(stderr, "Error closing pool directory '%s'. %s.\n", dir, strerror(errno));
exit(EXIT_FAILURE);
}
return ignored;
}
/**
* Creates a link to the specified disk entry.
*/
static void make_link(const char* pool_dir, struct snapraid_disk* disk, const char* sub)
{
char path[PATH_MAX];
char linkto[PATH_MAX];
int ret;
/* make the paths */
pathprint(path, sizeof(path), "%s%s", pool_dir, sub);
pathprint(linkto, sizeof(linkto), "%s%s", disk->dir, sub);
/* create the ancestor directories */
ret = mkancestor(path);
if (ret != 0) {
exit(EXIT_FAILURE);
}
/* create the symlink */
ret = symlink(linkto, path);
if (ret != 0) {
if (errno == EEXIST) {
fprintf(stderr, "warning: '%s' already exists\n", path);
#ifdef _WIN32
} else if (errno == EPERM) {
fprintf(stderr, "You must run as Adminstrator to be able to create symlinks.\n");
exit(EXIT_FAILURE);
#endif
} else {
fprintf(stderr, "Error writing symlink '%s'. %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);
}
}
}
void state_pool(struct snapraid_state* state)
{
tommy_node* i;
char pool_dir[PATH_MAX];
int ret;
unsigned count;
if (state->pool[0] == 0) {
fprintf(stderr, "To use the 'pool' command you must set the pool directory in the configuration file\n");
exit(EXIT_FAILURE);
}
/* pool directory with final slash */
pathprint(pool_dir, sizeof(pool_dir), "%s", state->pool);
pathslash(pool_dir, sizeof(pool_dir));
/* create it if missing */
ret = mkancestor(pool_dir);
if (ret != 0) {
exit(EXIT_FAILURE);
}
/* first clear the previous pool tree */
clean_dir(state, pool_dir);
/* for each disk */
count = 0;
for(i=state->disklist;i!=0;i=i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
/* for each file */
for(j=disk->filelist;j!=0;j=j->next) {
struct snapraid_file* file = j->data;
make_link(pool_dir, disk, file->sub);
++count;
}
/* for each link */
for(j=disk->linklist;j!=0;j=j->next) {
struct snapraid_link* link = j->data;
make_link(pool_dir, disk, link->sub);
++count;
}
/* we ignore empty dirs in disk->dir */
}
if (!state->gui) {
if (count)
printf("%u links created\n", count);
else
printf("No link created\n");
}
}

View file

@ -30,7 +30,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG]
.PD 0
.PP
.PD
sync|diff|dup|check|fix
sync|pool|diff|dup|check|fix
.PD 0
.PP
.PD
@ -114,7 +114,7 @@ With a Backup you are able to recover from a complete
failure of the whole disk array.
.IP \(bu
Only file, timestamps, symlinks and hardlinks are saved.
Permissions, ownership, extended attributes are not saved.
Permissions, ownership and extended attributes are not saved.
.PD
.SH GETTING STARTED
To use SnapRAID you need to first select one disk of your disk array
@ -250,7 +250,37 @@ When this command completes, your data is SAFE.
.PP
At this point you can start using your array as you like, and periodically
update the redundancy information running the \(a"sync\(a" command.
.SS Pooling
To have all the files in your array shown in the same directory tree,
you can enable \(a"pooling\(a", that consists in creating a virtual view of all
the files in your array using symbolic links.
You can configure the \(a"pooling\(a" directory in the configuration file with:
.PP
.RS 4
pool /pool
.PD 0
.PP
.PD
.RE
.PP
or, if you are in Windows, with:
.PP
.RS 4
pool C:\(rspool
.PD 0
.PP
.PD
.RE
.PP
and then run the \(a"pool\(a" command.
.PP
.RS 4
snapraid pool
.PD 0
.PP
.PD
.RE
.SS Checking & Fixing
To check the integrity of your data you can use the \(a"check\(a" command:
.PP
.RS 4
@ -275,7 +305,7 @@ Note that the fix command will revert your data at the state of the
last \(a"sync\(a" command executed. It works like a snapshot was taken
in \(a"sync\(a".
.PP
In this regard snapraid is more like a backup program than a RAID
In this regard SnapRAID is more like a backup program than a RAID
system. For example, you can use it to recover from an accidentally
deleted directory, simply running the fix command like.
.PP
@ -322,21 +352,33 @@ without losing the work already done.
.PP
The \(a"content\(a", \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary.
The files in the array are NOT modified.
.SS pool
Creates or updates in the \(a"pooling\(a" directory a virtual view of all
the files of your disk array.
.PP
The files are not really copied here, but just linked using
symbolic links.
.PP
When updating, all the present symbolic links and empty
subdirectories are deleted and replaced with the new
view of the array. Any othe regular file is left in place.
.PP
Nothing is modified outside the pool directory.
.SS diff
Lists all the files modified from the last \(a"sync\(a" command that
have to recompute their redundancy data.
.PP
Nothing is modified.
.SS dup
Lists all the duplicate files. Two files are assumed equal if their hashes
are matching. The effective data is not read.
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
.PP
Nothing is modified.
.SS check
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
in the previous \(a"sync\(a" command.
If you use the \(hyA, \(hy\(hyaudit\(hyonly option, only the file
If you use the \(hya, \(hy\(hyaudit\(hyonly option, only the file
data is checked, and the redundandy data is ignored.
.PP
Files are identified by path, and checked by content.
@ -381,6 +423,8 @@ commands.
Only the files present in the specified disk are processed.
You must specify a disk name as named in the configuration
file.
In \(a"check\(a", you can make it faster, specifing also \(hya, \(hy\(hyaudit\(hyonly
option, to avoid to access other disks to check parity data.
If you combine \(hy\(hyfilter and \(hy\(hyfilter\(hydisk options, only files
matching both the two groups of rules are selected.
This option can be used many times.
@ -405,9 +449,6 @@ size and time with second precision is assumed identical at
the previous one, and not resynched.
The nanosecond part of timestamps is ignored because most of
copy programs are not able to correctly restore it.
Note that if you use this option, all the hard\(hylinks will be
stored independently wasting space. So, if you use hard\(hylinks,
it\(cqs better to don\(cqt use it.
This option can be used only with the \(a"sync\(a" and \(a"diff\(a" commands.
.TP
.B \(hyZ, \(hy\(hyforce\(hyzero
@ -503,7 +544,7 @@ keep the NAME fixed.
You should use one option for each disk of the array.
.SS nohidden
Excludes all the hidden files and directory.
In Unix hidden files are the ones starting with \(cq.\(cq.
In Unix hidden files are the ones starting with \(a".\(a".
In Windows they are the ones with the hidden attribute.
.SS exclude/include PATTERN
Defines the file or directory patterns to exclude and include
@ -554,11 +595,14 @@ going to waste 1.2 GiB.
.SS autosave SIZE_IN_GIBIBYTES
Automatically save the state when synching after the specied amount
of GiB processed.
This option is useful to avoid to restart from scratch long \(cqsync\(cq
This option is useful to avoid to restart from scratch long \(a"sync\(a"
commands interrupted by a machine crash, or any other event that
may interrupt SnapRAID.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
.SS pool DIR
Defines the pooling directory where the virtual view of the disk
array is created using the \(a"pool\(a" command.
.SS Examples
An example of a typical configuration for Unix is:
.PP
@ -865,7 +909,7 @@ As paranoid check, you can now run a whole \(a"check\(a" command to ensure that
everything is OK.
.PP
.RS 4
snapraid \(hyd NAME check
snapraid \(hyd NAME \(hya check
.PD 0
.PP
.PD
@ -873,6 +917,9 @@ snapraid \(hyd NAME check
.PP
Where NAME is the name of the disk, like \(a"d1\(a" as in our previous example.
.PP
The options \(hyd and \(hya tell SnapRAID to check only the specified disk,
and ignore all the other data and redundancy disks.
.PP
This command will take a long time.
.SS STEP 4 \(hy> Sync
Run the \(a"sync\(a" command to resyncronize the array with the new disk.

View file

@ -122,6 +122,7 @@ void signal_handler(int signal)
#define OPERATION_FIX 3
#define OPERATION_DRY 4
#define OPERATION_DUP 5
#define OPERATION_POOL 6
int main(int argc, char* argv[])
{
@ -276,6 +277,8 @@ int main(int argc, char* argv[])
operation = OPERATION_DRY;
} else if (strcmp(argv[optind], "dup") == 0) {
operation = OPERATION_DUP;
} else if (strcmp(argv[optind], "pool") == 0) {
operation = OPERATION_POOL;
} else {
fprintf(stderr, "Unknown command '%s'\n", argv[optind]);
exit(EXIT_FAILURE);
@ -386,6 +389,10 @@ int main(int argc, char* argv[])
state_read(&state);
state_dup(&state);
} else if (operation == OPERATION_POOL) {
state_read(&state);
state_pool(&state);
} else {
state_read(&state);

View file

@ -56,3 +56,11 @@ block_size 256
# Format: "autosave_size SIZE_IN_GiB"
# autosize 500
# Defines the pooling directory where the virtual view of the disk
# array is created using the "pool" command.
# The files are not really copied here, but just linked using
# symbolic links.
# This directory must be outside the array.
# Format: "pool DIR"
#pool /pool

View file

@ -57,3 +57,11 @@ block_size 256
# Format: "autosave_size SIZE_IN_GiB"
# autosize 500
# Defines the pooling directory where the virtual view of the disk
# array is created using the "pool" command.
# The files are not really copied here, but just linked using
# symbolic links.
# This directory must be outside the array.
# Format: "pool DIR"
#pool C:\pool

View file

@ -9,7 +9,7 @@ Synopsis
: [-Z, --force-zero] [-E, --force-empty]
: [-s, --start BLKSTART] [-t, --count BLKCOUNT]
: [-v, --verbose]
: sync|diff|dup|check|fix
: sync|pool|diff|dup|check|fix
:snapraid [-V, --version] [-h, --help]
@ -71,7 +71,7 @@ Limitations
With a Backup you are able to recover from a complete
failure of the whole disk array.
* Only file, timestamps, symlinks and hardlinks are saved.
Permissions, ownership, extended attributes are not saved.
Permissions, ownership and extended attributes are not saved.
Getting Started
To use SnapRAID you need to first select one disk of your disk array
@ -143,6 +143,23 @@ Getting Started
At this point you can start using your array as you like, and periodically
update the redundancy information running the "sync" command.
Pooling
To have all the files in your array shown in the same directory tree,
you can enable "pooling", that consists in creating a virtual view of all
the files in your array using symbolic links.
You can configure the "pooling" directory in the configuration file with:
:pool /pool
or, if you are in Windows, with:
:pool C:\pool
and then run the "pool" command.
:snapraid pool
Checking & Fixing
To check the integrity of your data you can use the "check" command:
:snapraid check
@ -157,7 +174,7 @@ Getting Started
last "sync" command executed. It works like a snapshot was taken
in "sync".
In this regard snapraid is more like a backup program than a RAID
In this regard SnapRAID is more like a backup program than a RAID
system. For example, you can use it to recover from an accidentally
deleted directory, simply running the fix command like.
@ -192,6 +209,19 @@ Commands
The "content", "parity" and "q-parity" files are modified if necessary.
The files in the array are NOT modified.
pool
Creates or updates in the "pooling" directory a virtual view of all
the files of your disk array.
The files are not really copied here, but just linked using
symbolic links.
When updating, all the present symbolic links and empty
subdirectories are deleted and replaced with the new
view of the array. Any othe regular file is left in place.
Nothing is modified outside the pool directory.
diff
Lists all the files modified from the last "sync" command that
have to recompute their redundancy data.
@ -199,8 +229,8 @@ Commands
Nothing is modified.
dup
Lists all the duplicate files. Two files are assumed equal if their hashes
are matching. The effective data is not read.
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
Nothing is modified.
@ -208,7 +238,7 @@ Commands
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
in the previous "sync" command.
If you use the -A, --audit-only option, only the file
If you use the -a, --audit-only option, only the file
data is checked, and the redundandy data is ignored.
Files are identified by path, and checked by content.
@ -255,6 +285,8 @@ Options
Only the files present in the specified disk are processed.
You must specify a disk name as named in the configuration
file.
In "check", you can make it faster, specifing also -a, --audit-only
option, to avoid to access other disks to check parity data.
If you combine --filter and --filter-disk options, only files
matching both the two groups of rules are selected.
This option can be used many times.
@ -380,7 +412,7 @@ Configuration
nohidden
Excludes all the hidden files and directory.
In Unix hidden files are the ones starting with '.'.
In Unix hidden files are the ones starting with ".".
In Windows they are the ones with the hidden attribute.
exclude/include PATTERN
@ -431,12 +463,16 @@ Configuration
autosave SIZE_IN_GIBIBYTES
Automatically save the state when synching after the specied amount
of GiB processed.
This option is useful to avoid to restart from scratch long 'sync'
This option is useful to avoid to restart from scratch long "sync"
commands interrupted by a machine crash, or any other event that
may interrupt SnapRAID.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
pool DIR
Defines the pooling directory where the virtual view of the disk
array is created using the "pool" command.
Examples
An example of a typical configuration for Unix is:
@ -554,7 +590,7 @@ Recovering
Then proceed with the following steps.
STEP 1 -> Reconfigure
STEP 1 -> Reconfigure
You need some space to recover, even better if you already have an additional
disk, but in case, also an external USB or remote one is enough.
@ -570,7 +606,7 @@ Recovering
:disk d1 /mnt/new_spare_disk/
STEP 2 -> Fix
STEP 2 -> Fix
Run the fix command, storing the log in an external file with:
:snapraid -d NAME fix 2>fix.log
@ -596,17 +632,20 @@ Recovering
but take care that after synching you will no more able to retry the
"fix" command!
STEP 3 -> Check
STEP 3 -> Check
As paranoid check, you can now run a whole "check" command to ensure that
everything is OK.
:snapraid -d NAME check
:snapraid -d NAME -a check
Where NAME is the name of the disk, like "d1" as in our previous example.
The options -d and -a tell SnapRAID to check only the specified disk,
and ignore all the other data and redundancy disks.
This command will take a long time.
STEP 4 -> Sync
STEP 4 -> Sync
Run the "sync" command to resyncronize the array with the new disk.
To avoid a long time sync you can use the "--find-by-name" option to

View file

@ -13,7 +13,7 @@ snapraid [-c, --conf CONFIG]
[-Z, --force-zero] [-E, --force-empty]
[-s, --start BLKSTART] [-t, --count BLKCOUNT]
[-v, --verbose]
sync|diff|dup|check|fix
sync|pool|diff|dup|check|fix
snapraid [-V, --version] [-h, --help]
@ -81,7 +81,7 @@ Other limitations are:
With a Backup you are able to recover from a complete
failure of the whole disk array.
* Only file, timestamps, symlinks and hardlinks are saved.
Permissions, ownership, extended attributes are not saved.
Permissions, ownership and extended attributes are not saved.
4 GETTING STARTED
@ -156,6 +156,27 @@ When this command completes, your data is SAFE.
At this point you can start using your array as you like, and periodically
update the redundancy information running the "sync" command.
4.1 Pooling
-----------
To have all the files in your array shown in the same directory tree,
you can enable "pooling", that consists in creating a virtual view of all
the files in your array using symbolic links.
You can configure the "pooling" directory in the configuration file with:
pool /pool
or, if you are in Windows, with:
pool C:\pool
and then run the "pool" command.
snapraid pool
4.2 Checking & Fixing
---------------------
To check the integrity of your data you can use the "check" command:
snapraid check
@ -170,7 +191,7 @@ Note that the fix command will revert your data at the state of the
last "sync" command executed. It works like a snapshot was taken
in "sync".
In this regard snapraid is more like a backup program than a RAID
In this regard SnapRAID is more like a backup program than a RAID
system. For example, you can use it to recover from an accidentally
deleted directory, simply running the fix command like.
@ -210,7 +231,22 @@ without losing the work already done.
The "content", "parity" and "q-parity" files are modified if necessary.
The files in the array are NOT modified.
5.2 diff
5.2 pool
--------
Creates or updates in the "pooling" directory a virtual view of all
the files of your disk array.
The files are not really copied here, but just linked using
symbolic links.
When updating, all the present symbolic links and empty
subdirectories are deleted and replaced with the new
view of the array. Any othe regular file is left in place.
Nothing is modified outside the pool directory.
5.3 diff
--------
Lists all the files modified from the last "sync" command that
@ -218,27 +254,27 @@ have to recompute their redundancy data.
Nothing is modified.
5.3 dup
5.4 dup
-------
Lists all the duplicate files. Two files are assumed equal if their hashes
are matching. The effective data is not read.
Lists all the duplicate files. Two files are assumed equal if their
hashes are matching. The effective data is not read.
Nothing is modified.
5.4 check
5.5 check
---------
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
in the previous "sync" command.
If you use the -A, --audit-only option, only the file
If you use the -a, --audit-only option, only the file
data is checked, and the redundandy data is ignored.
Files are identified by path, and checked by content.
Nothing is modified.
5.5 fix
5.6 fix
-------
Checks and fix all the files. It's like "check" but it
@ -284,6 +320,8 @@ SnapRAID provides the following options:
Only the files present in the specified disk are processed.
You must specify a disk name as named in the configuration
file.
In "check", you can make it faster, specifing also -a, --audit-only
option, to avoid to access other disks to check parity data.
If you combine --filter and --filter-disk options, only files
matching both the two groups of rules are selected.
This option can be used many times.
@ -308,9 +346,6 @@ SnapRAID provides the following options:
the previous one, and not resynched.
The nanosecond part of timestamps is ignored because most of
copy programs are not able to correctly restore it.
Note that if you use this option, all the hard-links will be
stored independently wasting space. So, if you use hard-links,
it's better to don't use it.
This option can be used only with the "sync" and "diff" commands.
-Z, --force-zero
@ -425,7 +460,7 @@ You should use one option for each disk of the array.
------------
Excludes all the hidden files and directory.
In Unix hidden files are the ones starting with '.'.
In Unix hidden files are the ones starting with ".".
In Windows they are the ones with the hidden attribute.
7.6 exclude/include PATTERN
@ -480,12 +515,16 @@ going to waste 1.2 GiB.
---- 7.7.1 autosave SIZE_IN_GIBIBYTES ----
Automatically save the state when synching after the specied amount
of GiB processed.
This option is useful to avoid to restart from scratch long 'sync'
This option is useful to avoid to restart from scratch long "sync"
commands interrupted by a machine crash, or any other event that
may interrupt SnapRAID.
The SIZE argument is specified in gibibytes. Where one gibi bytes
is 1073741824 bytes.
---- 7.7.2 pool DIR ----
Defines the pooling directory where the virtual view of the disk
array is created using the "pool" command.
7.8 Examples
------------
@ -611,7 +650,9 @@ scheduled SnapRAID nightly sync.
Then proceed with the following steps.
---- 9.0.1 STEP 1 -> Reconfigure ----
9.1 STEP 1 -> Reconfigure
-------------------------
You need some space to recover, even better if you already have an additional
disk, but in case, also an external USB or remote one is enough.
@ -627,7 +668,9 @@ to:
disk d1 /mnt/new_spare_disk/
---- 9.0.2 STEP 2 -> Fix ----
9.2 STEP 2 -> Fix
-----------------
Run the fix command, storing the log in an external file with:
snapraid -d NAME fix 2>fix.log
@ -653,17 +696,24 @@ If you are satisfied of the recovering, you can now proceed further,
but take care that after synching you will no more able to retry the
"fix" command!
---- 9.0.3 STEP 3 -> Check ----
9.3 STEP 3 -> Check
-------------------
As paranoid check, you can now run a whole "check" command to ensure that
everything is OK.
snapraid -d NAME check
snapraid -d NAME -a check
Where NAME is the name of the disk, like "d1" as in our previous example.
The options -d and -a tell SnapRAID to check only the specified disk,
and ignore all the other data and redundancy disks.
This command will take a long time.
---- 9.0.4 STEP 4 -> Sync ----
9.4 STEP 4 -> Sync
------------------
Run the "sync" command to resyncronize the array with the new disk.
To avoid a long time sync you can use the "--find-by-name" option to

67
state.c
View file

@ -38,6 +38,8 @@ void state_init(struct snapraid_state* state)
state->parity_device = 0;
state->qarity[0] = 0;
state->qarity_device = 0;
state->pool[0] = 0;
state->pool_device = 0;
state->level = 1; /* default is the lowest protection */
state->hash = HASH_MURMUR3; /* default is the fastest */
tommy_list_init(&state->disklist);
@ -123,6 +125,11 @@ static void state_config_check(struct snapraid_state* state, const char* path, i
exit(EXIT_FAILURE);
}
if (state->pool[0] != 0 && disk->device == state->pool_device) {
fprintf(stderr, "Disk '%s' and pool '%s' are on the same device.\n", disk->dir, state->pool);
exit(EXIT_FAILURE);
}
++diskcount;
}
@ -148,12 +155,30 @@ static void state_config_check(struct snapraid_state* state, const char* path, i
fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
}
if (state->pool[0] != 0 && state->pool_device == 0) {
fprintf(stderr, "Disk '%s' has a zero serial number.\n", state->pool);
fprintf(stderr, "This is not necessarely wrong, but for using SnapRAID\n");
fprintf(stderr, "it's better to change the serial number of the disk.\n");
fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
}
#endif
if (state->qarity[0] != 0 && state->parity_device == state->qarity_device) {
fprintf(stderr, "Parity '%s' and '%s' are on the same device.\n", state->parity, state->qarity);
exit(EXIT_FAILURE);
}
if (state->pool[0] != 0 && state->pool_device == state->parity_device) {
fprintf(stderr, "Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity);
exit(EXIT_FAILURE);
}
if (state->pool[0] != 0 && state->qarity[0] != 0 && state->pool_device == state->qarity_device) {
fprintf(stderr, "Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->qarity);
exit(EXIT_FAILURE);
}
}
/* count the content files */
@ -283,7 +308,7 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
pathimport(state->parity, sizeof(state->parity), buffer);
/* get the device of the directory containing the content file */
/* get the device of the directory containing the parity file */
pathimport(device, sizeof(device), buffer);
slash = strrchr(device, '/');
if (slash)
@ -319,7 +344,7 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
pathimport(state->qarity, sizeof(state->qarity), buffer);
/* get the device of the directory containing the content file */
/* get the device of the directory containing the qparity file */
pathimport(device, sizeof(device), buffer);
slash = strrchr(device, '/');
if (slash)
@ -335,6 +360,42 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
/* we have two level of parity */
state->level = 2;
} else if (strcmp(tag, "pool") == 0) {
char device[PATH_MAX];
char* slash;
struct stat st;
if (*state->pool) {
fprintf(stderr, "Multiple 'pool' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'pool' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
if (!*buffer) {
fprintf(stderr, "Empty 'pool' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
pathimport(state->pool, sizeof(state->pool), buffer);
/* get the device of the directory containing the pool tree */
pathimport(device, sizeof(device), buffer);
slash = strrchr(device, '/');
if (slash)
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) != 0) {
fprintf(stderr, "Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", device, path, line);
exit(EXIT_FAILURE);
}
state->pool_device = st.st_dev;
} else if (strcmp(tag, "content") == 0) {
struct snapraid_content* content;
char device[PATH_MAX];
@ -528,6 +589,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
fprintf(stderr, "parity:%s\n", state->parity);
if (state->qarity[0] != 0)
fprintf(stderr, "qarity:%s\n", state->qarity);
if (state->pool[0] != 0)
fprintf(stderr, "pool:%s\n", state->pool);
fflush(stderr);
}
}

View file

@ -42,6 +42,8 @@ struct snapraid_state {
uint64_t parity_device; /**< Device identifier of the parity. */
char qarity[PATH_MAX]; /**< Path of the qarity file. */
uint64_t qarity_device; /**< Device identifier of the qarity. */
char pool[PATH_MAX]; /**< Path of the pool tree. */
uint64_t pool_device; /**< Device identifier of the pool. */
unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */
unsigned hash; /**< Hash kind used. */
tommy_list contentlist; /**< List of content files. */
@ -103,10 +105,15 @@ void state_check(struct snapraid_state* state, int check, int fix, block_off_t b
void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount);
/**
* Find duplicates.
* Finds duplicates.
*/
void state_dup(struct snapraid_state* state);
/**
* Creates pool tree.
*/
void state_pool(struct snapraid_state* state);
/**
* Filter files, symlinks and dirs.
* Apply an additional filter to the list currently loaded.

View file

@ -6,3 +6,4 @@ disk disk1 test/disk1/
disk disk2 test/disk2/
disk disk3 test/disk3/
exclude *.unrecoverable
pool test/pool