rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Extend --force-device to ignore more erroneous conditions

Browse source 
We want to allow fix also if some disk is unaccessible,
and content files not present.
This commit is contained in:
Andrea Mazzoleni 2015-10-16 23:17:38 +02:00
parent 841e6306e3
commit 3e83b15bbb
12 changed files with 223 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

3
HISTORY
View file

@ -20,6 +20,9 @@ SnapRAID HISTORY
uses 17 bytes of memory, instead of the previous 28 bytes
(for 32 bit) or 36 bytes (for 64 bit).
This could result is a memory saving of up the 50%.
* Extends the -D, --force-device option to ignore more
erroneous conditions in the 'fix' command, like unaccessible
disks, or disks sharing the same physical device.
* Extends the -h, --pre-hash option to also verify moved and
copied files into the array before running a 'sync'.
* Update 'best' RAID functions for recent Atom CPUs.

9
TODO
View file

@ -5,15 +5,6 @@ This is the list of TODO items for SnapRAID.
- Next
* Improvement in fix, due missing disks, when using -d DISK.
A. Don't halt due to missing content files as long as there is one valid content file
B. Don't halt due to missing disks when enough parity exist to replace them.
See thread: https://sourceforge.net/p/snapraid/discussion/1677233/thread/b1387089/
Two disks died, like d5 and d6, but recovering with "-d d5" was not possible
because disk d6 disappeared, like I:\ drive missing, including its mount point and
content file. In Linux likely the mount point would be still present, but not in Windows.
The content file is instead missing in all platforms.
- Minor
* Restore ownership and permissions, at least in Unix.

6
cmdline/check.c
View file

@ -1927,6 +1927,12 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* if fixing, create the file and open for writing */
/* if it fails, we cannot continue */
for (l = 0; l < state->level; ++l) {
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access) {
parity_ptr[l] = 0;
continue;
}
parity_ptr[l] = &parity[l];
ret = parity_create(parity_ptr[l], state->parity[l].path, &out_size, state->file_mode);
if (ret == -1) {

3
cmdline/elem.c
View file

@ -712,7 +712,7 @@ int dir_name_compare(const void* void_arg, const void* void_data)
return strcmp(arg, dir->sub);
}
struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev)
struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, int skip)
{
struct snapraid_disk* disk;
@ -735,6 +735,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
disk->has_unsupported_uuid = 0;
disk->had_empty_uuid = 0;
disk->mapping_idx = -1;
disk->skip_access = skip;
tommy_list_init(&disk->filelist);
tommy_list_init(&disk->deletedlist);
tommy_hashdyn_init(&disk->inodeset);

4
cmdline/elem.h
View file

@ -339,6 +339,7 @@ struct snapraid_disk {
int has_unsupported_uuid; /**< If the disk doesn't report UUID, meaning it's not supported. */
int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
int skip_access; /**< If the disk is unaccessible and it should be skipped. */
/**
* Mapping of chunks in the parity.
@ -407,6 +408,7 @@ struct snapraid_parity {
uint64_t device; /**< Device identifier of the parity. */
block_off_t total_blocks; /**< Number of total blocks. */
block_off_t free_blocks; /**< Number of free blocks at the last sync. */
int skip_access; /**< If the disk is unaccessible and it should be skipped. */
/**
* Cumulative time used for parity disks.
@ -864,7 +866,7 @@ static inline tommy_uint32_t dir_name_hash(const char* name)
/**
* Allocate a disk.
*/
struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev);
struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev, int skip);
/**
* Deallocate a disk.

6
cmdline/search.c
View file

@ -282,7 +282,11 @@ void state_search_array(struct snapraid_state* state)
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
msg_progress("Scanning disk %s...\n", disk->name);
/* skip data disks that are not accessible */
if (disk->skip_access)
continue;
msg_progress("Searching disk %s...\n", disk->name);
search_dir(state, disk, disk->dir, "");
}

32
cmdline/snapraid.c
View file

@ -68,7 +68,7 @@ void usage(void)
printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n");
printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force synching of disks that get empty\n");
printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n");
printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands on disks with same device id\n");
printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands with unaccessible/shared disks\n");
printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n");
printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force commands requiring a full sync\n");
printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n");
@ -794,6 +794,18 @@ int main(int argc, char* argv[])
}
}
switch (operation) {
case OPERATION_FIX :
break;
default :
if (opt.force_device) {
/* LCOV_EXCL_START */
log_fatal("You cannot use -D, --force-device with the '%s' command\n", command);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
switch (operation) {
case OPERATION_SYNC :
case OPERATION_CHECK :
@ -915,6 +927,14 @@ int main(int argc, char* argv[])
break;
}
switch (operation) {
case OPERATION_FIX :
case OPERATION_CHECK :
/* avoid to stop processing if a content file is not accessible */
opt.skip_content_access = 1;
break;
}
switch (operation) {
case OPERATION_DIFF :
case OPERATION_LIST :
@ -968,7 +988,7 @@ int main(int argc, char* argv[])
#if HAVE_LOCKFILE
/* create the lock file */
if (!opt.skip_lock) {
if (!opt.skip_lock && state.lockfile[0]) {
lock = lock_lock(state.lockfile);
if (lock == -1) {
/* LCOV_EXCL_START */
@ -1089,7 +1109,8 @@ int main(int argc, char* argv[])
} else if (operation == OPERATION_DRY) {
state_read(&state);
/* apply the command line filter */
/* filter */
state_skip(&state);
state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing, filter_error);
memory();
@ -1189,7 +1210,8 @@ int main(int argc, char* argv[])
state_search_array(&state);
}
/* apply the command line filter */
/* filter */
state_skip(&state);
state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing, filter_error);
memory();
@ -1215,7 +1237,7 @@ int main(int argc, char* argv[])
log_close(log_file);
#if HAVE_LOCKFILE
if (!opt.skip_lock) {
if (!opt.skip_lock && state.lockfile[0]) {
if (lock_unlock(lock) == -1) {
/* LCOV_EXCL_START */
log_fatal("Error closing the lock file '%s'. %s.\n", state.lockfile, strerror(errno));

197
cmdline/state.c
View file

@ -134,6 +134,7 @@ void state_init(struct snapraid_state* state)
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
state->parity[l].tick = 0;
state->parity[l].skip_access = 0;
}
state->tick_io = 0;
state->tick_cpu = 0;
@ -225,6 +226,10 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
tommy_node* j;
struct snapraid_disk* disk = i->data;
/* skip data disks that are not accessible */
if (disk->skip_access)
continue;
#ifdef _WIN32
if (disk->device == 0) {
/* LCOV_EXCL_START */
@ -237,10 +242,15 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
#endif
if (!state->opt.force_device) {
for (j = i->next; j != 0; j = j->next) {
struct snapraid_disk* other = j->data;
if (disk->device == other->device) {
for (j = i->next; j != 0; j = j->next) {
struct snapraid_disk* other = j->data;
if (disk->device == other->device) {
if (state->opt.force_device) {
/* mark disks to be skipped */
disk->skip_access = 1;
other->skip_access = 1;
msg_progress("DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
} else {
/* LCOV_EXCL_START */
log_fatal("Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
#ifdef _WIN32
@ -258,6 +268,10 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
}
/* skip data disks that are not accessible */
if (disk->skip_access)
continue;
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
if (disk->device == state->parity[l].device) {
@ -290,8 +304,14 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
/* check device of parity disks */
if (!state->opt.skip_device && !state->opt.skip_parity_access) {
#ifdef _WIN32
for (l = 0; l < state->level; ++l) {
unsigned j;
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
#ifdef _WIN32
if (state->parity[l].device == 0) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].path);
@ -301,27 +321,37 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
#endif
for (l = 0; l < state->level; ++l) {
unsigned j;
for (j = l + 1; j < state->level; ++j) {
if (state->parity[l].device == state->parity[j].device) {
/* LCOV_EXCL_START */
log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].path, state->parity[j].path);
if (state->opt.force_device) {
/* mark disks to be skipped */
state->parity[l].skip_access = 1;
state->parity[j].skip_access = 1;
msg_progress("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
} else {
/* LCOV_EXCL_START */
log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].path, state->parity[j].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
for (l = 0; l < state->level; ++l) {
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
if (state->pool[0] != 0 && state->pool_device == state->parity[l].device) {
/* LCOV_EXCL_START */
log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].path);
@ -352,7 +382,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
#endif
/* count the content files */
if (!state->opt.skip_device) {
if (!state->opt.skip_device && !state->opt.skip_content_access) {
unsigned content_count;
content_count = 0;
@ -539,6 +569,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
} else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
char device[PATH_MAX];
char* slash;
uint64_t dev;
int skip_access;
if (*state->parity[level].path) {
/* LCOV_EXCL_START */
@ -564,6 +596,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
pathimport(state->parity[level].path, sizeof(state->parity[level].path), buffer);
skip_access = 0;
if (!state->opt.skip_parity_access) {
struct stat st;
@ -574,19 +607,36 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->parity[level].device = st.st_dev;
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
skip_access = 1;
msg_progress("DANGER! Skipping unaccessible parity disk '%s'...\n", tag);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* if parity is skipped, uses a fake device */
state->parity[level].device = 0;
/* use a fake device */
dev = 0;
}
state->parity[level].device = dev;
state->parity[level].skip_access = skip_access;
/* adjust the level */
if (state->level < level + 1)
state->level = level + 1;
@ -655,6 +705,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
char device[PATH_MAX];
char* slash;
struct stat st;
uint64_t dev;
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
@ -698,20 +749,28 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
if (state->opt.skip_content_access) {
/* use a fake device */
dev = 0;
msg_progress("WARNING! Skipping unaccessible content file '%s'...\n", buffer);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* set the lock file at the first content file */
if (tommy_list_empty(&state->contentlist)) {
/* set the lock file at the first accessible content file */
if (state->lockfile[0] == 0 && dev != 0) {
pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
}
content = content_alloc(buffer, st.st_dev);
content = content_alloc(buffer, dev);
tommy_list_insert_tail(&state->contentlist, &content->node, content);
} else if (strcmp(tag, "disk") == 0) {
@ -719,6 +778,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
char device[PATH_MAX];
struct snapraid_disk* disk;
uint64_t dev;
int skip_access;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
@ -768,23 +828,38 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* LCOV_EXCL_STOP */
}
/* if the disk has to be present */
skip_access = 0;
if (!state->opt.skip_disk_access) {
struct stat st;
if (stat(device, &st) != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
skip_access = 1;
msg_progress("DANGER! Skipping unaccessible data disk '%s'...\n", buffer);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
dev = st.st_dev;
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* if data disks are skipped, uses a fake device */
/* use a fake device */
dev = 0;
}
disk = disk_alloc(buffer, dir, dev);
disk = disk_alloc(buffer, dir, dev, skip_access);
tommy_list_insert_tail(&state->disklist, &disk->node, disk);
} else if (strcmp(tag, "smartctl") == 0) {
@ -1069,7 +1144,7 @@ static struct snapraid_disk* find_disk(struct snapraid_state* state, const char*
/* without a configuration file, add disks automatically */
struct snapraid_disk* disk;
disk = disk_alloc(name, "DUMMY/", -1);
disk = disk_alloc(name, "DUMMY/", -1, 0);
tommy_list_insert_tail(&state->disklist, &disk->node, disk);
@ -3124,6 +3199,38 @@ void state_write(struct snapraid_state* state)
state->checked_read = 0; /* what we wrote is not checked in read */
}
void state_skip(struct snapraid_state* state)
{
tommy_node* i;
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
if (!disk->skip_access)
continue;
/* for each file */
for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
struct snapraid_file* file = j->data;
file_flag_set(file, FILE_IS_EXCLUDED);
}
/* for each link */
for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
struct snapraid_link* slink = j->data;
link_flag_set(slink, FILE_IS_EXCLUDED);
}
/* for each dir */
for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
struct snapraid_dir* dir = j->data;
dir_flag_set(dir, FILE_IS_EXCLUDED);
}
}
}
void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
{
tommy_node* i;

7
cmdline/state.h
View file

@ -91,6 +91,7 @@ struct snapraid_option {
int skip_content_check; /**< Relax some content file checks. */
int skip_parity_access; /**< Skip the parity access for commands that don't need it. */
int skip_disk_access; /**< Skip the data disk access for commands that don't need it. */
int skip_content_access; /**< Skip the content access for commands that don't need it. */
int kill_after_sync; /**< Kill the process after sync without saving the final state. */
int force_murmur3; /**< Force Murmur3 choice. */
int force_spooky2; /**< Force Spooky2 choice. */
@ -280,6 +281,12 @@ void state_pool(struct snapraid_state* state);
*/
void state_refresh(struct snapraid_state* state);
/**
* Skip files, symlinks and dirs.
* Apply any skip access disk.
*/
void state_skip(struct snapraid_state* state);
/**
* Filter files, symlinks and dirs.
* Apply an additional filter to the list currently loaded.

13
snapraid.1
View file

@ -1061,13 +1061,12 @@ This option can be used only with \[dq]sync\[dq], \[dq]check\[dq] or
\[dq]fix\[dq].
.TP
.B \-D, \-\-force\-device
Forces the insecure operation of fixing with disks on the same
physical device.
If SnapRAID detects that some disks have the same device ID,
it stops proceeding, because it\'s not a supported configuration.
But it could happen that you want to temporarily restore a lost
disk in the free space left in an already used disk. and this
option allows you to continue anyway.
Forces the insecure operation of fixing with unaccessible disks,
or with disks on the same physical device.
Like if you lost two data disks, and you have a spare disk to recover
only the first one, and you want to ignore the second unaccessible disk.
Or if you want to recover a disk in the free space left in an
already used disk, sharing the same physical device.
This option can be used only with \[dq]fix\[dq].
.TP
.B \-N, \-\-force\-nocopy

13
snapraid.d
View file

@ -775,13 +775,12 @@ Options
"fix".
-D, --force-device
Forces the insecure operation of fixing with disks on the same
physical device.
If SnapRAID detects that some disks have the same device ID,
it stops proceeding, because it's not a supported configuration.
But it could happen that you want to temporarily restore a lost
disk in the free space left in an already used disk. and this
option allows you to continue anyway.
Forces the insecure operation of fixing with unaccessible disks,
or with disks on the same physical device.
Like if you lost two data disks, and you have a spare disk to recover
only the first one, and you want to ignore the second unaccessible disk.
Or if you want to recover a disk in the free space left in an
already used disk, sharing the same physical device.
This option can be used only with "fix".
-N, --force-nocopy

13
snapraid.txt
View file

@ -834,13 +834,12 @@ SnapRAID provides the following options:
"fix".
-D, --force-device
Forces the insecure operation of fixing with disks on the same
physical device.
If SnapRAID detects that some disks have the same device ID,
it stops proceeding, because it's not a supported configuration.
But it could happen that you want to temporarily restore a lost
disk in the free space left in an already used disk. and this
option allows you to continue anyway.
Forces the insecure operation of fixing with unaccessible disks,
or with disks on the same physical device.
Like if you lost two data disks, and you have a spare disk to recover
only the first one, and you want to ignore the second unaccessible disk.
Or if you want to recover a disk in the free space left in an
already used disk, sharing the same physical device.
This option can be used only with "fix".
-N, --force-nocopy