rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 02:49:02 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Preliminary support for multiple parity files

Browse source 
This allows to split parity in different partitions.
This commit is contained in:
Andrea Mazzoleni 2016-04-26 21:48:39 +02:00
parent 1ce7b61f29
commit f564e9b91a
11 changed files with 873 additions and 462 deletions
Download patch file Download diff file Expand all files Collapse all files

5
HISTORY
View file

@ -3,7 +3,10 @@ SnapRAID HISTORY
11.0 2016/05
============
* In Linux add the missing support for filesystems using null device
* Added support for splitting the parity in multiple partitions. You can
now specify multiple files for a single parity. As soon a file
cannot grow anymore, the next one starts growing.
* In Linux added the missing support for filesystems using null device
numbers, like Btrfs. Note that to have full support you need also
the 'libblkid' library, otherwise you won't get the UUIDs.
* In the Windows version allow to use the escape char '^' to handle

5
TODO
View file

@ -5,11 +5,6 @@ This is the list of TODO items for SnapRAID.
- Next
* Allow to specify more than one partition for each parity disk.
Checking the UUID, we can avoid that users mess-up with disk specifications.
See discussion at:
https://sourceforge.net/p/snapraid/discussion/1677233/thread/02b2a2ae/
- Minor
* Allow to have "parial" parity disks, smaller than required ?

7
cmdline/check.c
View file

@ -1913,7 +1913,6 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
{
block_off_t blockmax;
data_off_t size;
data_off_t out_size;
int ret;
struct snapraid_parity_handle parity[LEV_MAX];
struct snapraid_parity_handle* parity_ptr[LEV_MAX];
@ -1948,7 +1947,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
}
parity_ptr[l] = &parity[l];
ret = parity_create(parity_ptr[l], l, state->parity[l].path, &out_size, state->file_mode);
ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l));
@ -1956,7 +1955,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* LCOV_EXCL_STOP */
}
ret = parity_chsize(parity_ptr[l], size, &out_size, state->opt.skip_fallocate);
ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
@ -1969,7 +1968,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* it may fail if the file doesn't exist, in this case we continue to check the files */
for (l = 0; l < state->level; ++l) {
parity_ptr[l] = &parity[l];
ret = parity_open(parity_ptr[l], l, state->parity[l].path, state->file_mode);
ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
msg_status("No accessible %s file, only files will be checked.\n", lev_name(l));
/* continue anyway */

17
cmdline/device.c
View file

@ -922,19 +922,22 @@ void state_device(struct snapraid_state* state, int operation, tommy_list* filte
/* for all parities */
for (j = 0; j < state->level; ++j) {
devinfo_t* entry;
unsigned s;
if (filterlist_disk != 0 && filter_path(filterlist_disk, 0, lev_config_name(j), 0) != 0)
continue;
entry = calloc_nofail(1, sizeof(devinfo_t));
for (s = 0; s < state->parity[j].split_mac; ++s) {
entry = calloc_nofail(1, sizeof(devinfo_t));
entry->device = state->parity[j].device;
pathcpy(entry->name, sizeof(entry->name), lev_config_name(j));
pathcpy(entry->mount, sizeof(entry->mount), state->parity[j].path);
pathcpy(entry->smartctl, sizeof(entry->smartctl), state->parity[j].smartctl);
pathcut(entry->mount); /* remove the parity file */
entry->device = state->parity[j].split_map[s].device;
pathcpy(entry->name, sizeof(entry->name), lev_config_name(j));
pathcpy(entry->mount, sizeof(entry->mount), state->parity[j].split_map[s].path);
pathcpy(entry->smartctl, sizeof(entry->smartctl), state->parity[j].smartctl);
pathcut(entry->mount); /* remove the parity file */
tommy_list_insert_tail(&high, &entry->node, entry);
tommy_list_insert_tail(&high, &entry->node, entry);
}
}
if (state->opt.fake_device) {

2
cmdline/dry.c
View file

@ -417,7 +417,7 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t
/* open the file for reading */
/* it may fail if the file doesn't exist, in this case we continue to dry the files */
for (l = 0; l < state->level; ++l) {
ret = parity_open(&parity_handle[l], l, state->parity[l].path, state->file_mode);
ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to dry.\n", lev_name(l));

42
cmdline/elem.h
View file

@ -433,19 +433,51 @@ struct snapraid_map {
tommy_node node;
};
/**
* Max number of parity split.
*/
#define SPLIT_MAX 8
/**
* Invalid parity size.
*
* This value is used to identify new parities,
* like when you alter the configuration adding
* a new parity level, creating it with 'fix'.
* Given that 'fix' doesn't write the content file,
* the new size will be written only at the next
* 'sync'.
*/
#define PARITY_SIZE_INVALID -1ULL
/**
* Parity split.
*/
struct snapraid_split {
char path[PATH_MAX]; /**< Path of the parity file. */
char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
/**
* Size of the parity split.
* Only the latest not zero size is allowed to grow.
* If the value is unset, it's PARITY_SIZE_INVALID.
*/
uint64_t size;
uint64_t device; /**< Device identifier of the parity. */
};
/**
* Parity.
*/
struct snapraid_parity {
char path[PATH_MAX]; /**< Path of the parity file. */
char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */
unsigned split_mac; /**< Number of parity splits. */
char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
uint64_t device; /**< Device identifier of the parity. */
block_off_t total_blocks; /**< Number of total blocks. */
block_off_t free_blocks; /**< Number of free blocks at the last sync. */
int skip_access; /**< If the disk is inaccessible and it should be skipped. */
int is_excluded; /**< If the parity is excluded by disk filter. */
int skip_access; /**< If at least one of the parity disk is inaccessible and it should be skipped. */
uint64_t tick; /**< Usage time. */
uint64_t progress_tick[PROGRESS_MAX]; /**< Last cpu ticks of progress. */
unsigned cached; /**< Number of IO blocks cached. */

681
cmdline/parity.c
View file

@ -150,305 +150,511 @@ void parity_overflow(struct snapraid_state* state, data_off_t size)
}
}
int parity_create(struct snapraid_parity_handle* parity, unsigned level, const char* path, data_off_t* out_size, int mode)
void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size)
{
int ret;
int flags;
unsigned s;
data_off_t size;
parity->level = level;
pathcpy(parity->path, sizeof(parity->path), path);
/* now compute the size */
size = 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
/* opening in sequential mode in Windows */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
flags = O_RDWR | O_CREAT | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
parity->f = open(parity->path, flags, 0600);
if (parity->f == -1) {
/* LCOV_EXCL_START */
log_fatal("Error opening parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
/* check for validity */
if (split->st.st_size < 0)
break;
if ((uint64_t)split->st.st_size < split->size) {
/* this is a size error, so report only up to this size */
/* but don't fail the function */
size += split->st.st_size;
break;
}
/* ignore instead size bigger than expected */
size += split->size;
}
/* get the stat info */
ret = fstat(parity->f, &parity->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
*out_size = size;
}
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(parity->f, 0, 0, POSIX_FADV_SEQUENTIAL);
if (ret != 0) {
int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode)
{
unsigned s;
handle->level = level;
handle->split_mac = 0;
for (s = 0;s < parity->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
int flags;
/* mark it as initialized */
pathcpy(split->path, sizeof(split->path), parity->split_map[s].path);
split->f = -1;
split->size = parity->split_map[s].size;
++handle->split_mac;
/* opening in sequential mode in Windows */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
flags = O_RDWR | O_CREAT | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
split->f = open(split->path, flags, 0600);
if (split->f == -1) {
/* LCOV_EXCL_START */
log_fatal("Error advising parity file '%s'. %s.\n", parity->path, strerror(ret));
log_fatal("Error opening parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
#endif
/* get the size of the existing data */
parity->valid_size = parity->st.st_size;
*out_size = parity->st.st_size;
/* get the stat info */
ret = fstat(split->f, &split->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
return 0;
/**
* If the parity size is not yet set, set it now.
* This happens when expanding the number of parities.
*/
if (split->size == PARITY_SIZE_INVALID)
split->size = split->st.st_size;
bail:
/* LCOV_EXCL_START */
close(parity->f);
parity->f = -1;
parity->valid_size = 0;
return -1;
/* LCOV_EXCL_STOP */
}
int parity_chsize(struct snapraid_parity_handle* parity, data_off_t size, data_off_t* out_size, int skip_fallocate)
{
int ret;
if (parity->st.st_size < size) {
int f_ret;
int f_errno;
const char* call;
#if HAVE_FALLOCATE
call = "fallocate";
if (!skip_fallocate) {
/* allocate real space using the specific Linux fallocate() operation. */
/* If the underline file-system doesn't support it, this operation fails, */
/* instead posix_fallocate() fallbacks to write the whole file. */
ret = fallocate(parity->f, 0, 0, size);
/* fallocate() returns the error number as positive integer, */
/* and in this case it doesn't set errno, just like posix_fallocate() */
/* Checking the glibc code (2.11.1 and 2.14.1) it seems that ENOSYS */
/* may be returned in errno, so we support both the return way */
if (ret > 0) { /* if a positive error is returned, convert it to errno */
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(split->f, 0, 0, POSIX_FADV_SEQUENTIAL);
if (ret != 0) {
/* LCOV_EXCL_START */
errno = ret;
ret = -1;
log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(ret));
goto bail;
/* LCOV_EXCL_STOP */
}
} else {
errno = EOPNOTSUPP;
ret = -1;
}
/* we get EOPNOTSUPP if the operation is not supported, like in ext3/ext2 */
/* or ENOSYS with kernel before 2.6.23 */
if (errno == EOPNOTSUPP || errno == ENOSYS) {
/* fallback using ftruncate() */
call = "ftruncate";
ret = ftruncate(parity->f, size);
}
#else
(void)skip_fallocate; /* avoid the warning */
/* allocate using a sparse file */
call = "ftruncate";
ret = ftruncate(parity->f, size);
#endif
/* save the state of the grow operation */
f_ret = ret;
f_errno = errno;
/* get the stat info */
ret = fstat(parity->f, &parity->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* return the new size */
*out_size = parity->st.st_size;
/* now check the error */
if (f_ret != 0) {
/* LCOV_EXCL_START */
if (f_errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' to size %" PRIu64 " using %s due lack of space.\n", parity->path, size, call);
} else {
log_fatal("Error growing parity file '%s' to size %" PRIu64 " using %s. Do you have enough space? %s.\n", parity->path, size, call, strerror(f_errno));
}
goto bail;
/* LCOV_EXCL_STOP */
}
} else if (parity->st.st_size > size) {
int f_ret;
int f_errno;
/* truncate the parity file */
ret = ftruncate(parity->f, size);
/* save the state of the shrink operation */
f_ret = ret;
f_errno = errno;
/* get the stat info */
ret = fstat(parity->f, &parity->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* return the new size */
*out_size = parity->st.st_size;
/* now check the error */
if (f_ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error truncating parity file '%s' to size %" PRIu64 ". %s.\n", parity->path, size, strerror(f_errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* adjust the valid to the new size */
parity->valid_size = size;
}
return 0;
bail:
/* LCOV_EXCL_START */
parity->valid_size = 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
close(split->f);
split->f = -1;
}
return -1;
/* LCOV_EXCL_STOP */
}
int parity_open(struct snapraid_parity_handle* parity, unsigned level, const char* path, int mode)
static int parity_handle_grow(struct snapraid_split_handle* split, data_off_t size, int skip_fallocate)
{
int ret;
int flags;
parity->level = level;
pathcpy(parity->path, sizeof(parity->path), path);
#if HAVE_FALLOCATE
if (!skip_fallocate) {
/* allocate real space using the specific Linux fallocate() operation. */
/* If the underline file-system doesn't support it, this operation fails, */
/* instead posix_fallocate() fallbacks to write the whole file. */
ret = fallocate(split->f, 0, 0, size);
/* open for read */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
/* O_NOATIME: do not change access time */
flags = O_RDONLY | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
parity->f = open_noatime(parity->path, flags);
if (parity->f == -1) {
log_fatal("Error opening parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* fallocate() returns the error number as positive integer, */
/* and in this case it doesn't set errno, just like posix_fallocate() */
/* Checking the glibc code (2.11.1 and 2.14.1) it seems that ENOSYS */
/* may be returned in errno, so we support both the return way */
if (ret > 0) { /* if a positive error is returned, convert it to errno */
/* LCOV_EXCL_START */
errno = ret;
ret = -1;
/* LCOV_EXCL_STOP */
}
} else {
errno = EOPNOTSUPP;
ret = -1;
}
/* we get EOPNOTSUPP if the operation is not supported, like in ext3/ext2 */
/* or ENOSYS with kernel before 2.6.23 */
if (errno == EOPNOTSUPP || errno == ENOSYS) {
/* fallback using ftruncate() */
ret = ftruncate(split->f, size);
}
#else
(void)skip_fallocate; /* avoid the warning */
/* allocate using a sparse file */
ret = ftruncate(split->f, size);
#endif
return ret;
}
/**
* Get the highest bit set.
*/
uint64_t hbit_u64(uint64_t v)
{
unsigned ilog;
ilog = 0;
while ((v /= 2) != 0)
++ilog;
return 1ULL << ilog;
}
static int parity_handle_fill(struct snapraid_split_handle* split, data_off_t size, uint32_t block_size, int skip_fallocate)
{
data_off_t base;
data_off_t delta;
data_off_t block_mask;
/* mask of bits used by the block size */
block_mask = ((data_off_t)block_size) - 1;
/* present size */
base = split->st.st_size;
/* truncate it to block size multiplier */
/* in case of damage the size may get wrong */
base &= ~block_mask;
/* size we have to increase */
delta = size - base;
/* grow the size one bit at time, like a kind of binary search */
while (delta != 0) {
int ret;
data_off_t run = hbit_u64(delta);
ret = parity_handle_grow(split, base + run, skip_fallocate);
if (ret != 0) {
/* we cannot grow, fallback to the smaller bits */
delta = run - 1;
/* mask out the block size */
delta &= ~block_mask;
} else {
/* mask out this bit */
delta &= ~run;
base += run;
}
}
/* reallocate the expected size */
/* if this fail, is there a problem */
return parity_handle_grow(split, base, skip_fallocate);
}
static int parity_handle_shrink(struct snapraid_split_handle* split, data_off_t size)
{
return ftruncate(split->f, size);
}
static int parity_handle_chsize(struct snapraid_split_handle* split, data_off_t size, uint32_t block_size, int skip_fallocate)
{
int ret;
int f_ret;
int f_errno;
int f_dir;
if (split->st.st_size < size) {
f_ret = parity_handle_fill(split, size, block_size, skip_fallocate);
f_errno = errno;
f_dir = 1;
} else if (split->st.st_size > size) {
f_ret = parity_handle_shrink(split, size);
f_errno = errno;
f_dir = -1;
} else {
f_ret = 0;
f_errno = 0;
f_dir = 0;
}
/* get the stat info */
ret = fstat(parity->f, &parity->st);
ret = fstat(split->f, &split->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
log_fatal("Error accessing parity file '%s'. %s.\n", split->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
/* get the size of the existing data */
parity->valid_size = parity->st.st_size;
/* now check the error */
if (f_ret != 0) {
/* LCOV_EXCL_START */
if (f_dir > 0) {
if (f_errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' to size %" PRIu64 " due lack of space.\n", split->path, size);
} else {
log_fatal("Error growing parity file '%s' to size %" PRIu64 ". Do you have enough space? %s.\n", split->path, size, strerror(f_errno));
}
} else {
log_fatal("Error truncating parity file '%s' to size %" PRIu64 ". %s.\n", split->path, size, strerror(f_errno));
}
return -1;
/* LCOV_EXCL_STOP */
}
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(parity->f, 0, 0, POSIX_FADV_SEQUENTIAL);
return 0;
}
static int parity_split_is_fixed(struct snapraid_parity_handle* handle, unsigned s)
{
/* next one */
++s;
/* the latest one is always growing */
if (s >= handle->split_mac)
return 0;
/* if the next it's 0, this one is growing */
if (handle->split_map[s].size == 0)
return 0;
return 1;
}
int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* parity, int* is_modified, data_off_t size, uint32_t block_size, int skip_fallocate)
{
int ret;
unsigned s;
if (size < 0)
return -1;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int is_fixed = parity_split_is_fixed(handle, s);
data_off_t run;
if (is_fixed) {
if ((uint64_t)size <= split->size) {
is_fixed = 0; /* not fixed anymore */
run = size;
} else {
run = split->size;
}
} else {
run = size;
}
ret = parity_handle_chsize(split, run, block_size, skip_fallocate);
if (ret != 0)
return -1;
if (split->st.st_size > run) {
/* LCOV_EXCL_START */
log_fatal("Unexpected over resizing parity file '%s' to size %" PRIu64 " resulting in size %" PRIu64 ".\n", split->path, run, split->st.st_size);
return -1;
/* LCOV_EXCL_STOP */
} else if (is_fixed && split->st.st_size < run) {
/* LCOV_EXCL_START */
log_fatal("Failed restoring parity file '%s' to size %" PRIu64 " resulting in size %" PRIu64 ".\n", split->path, run, split->st.st_size);
return -1;
/* LCOV_EXCL_STOP */
} else {
size -= split->st.st_size;
split->size = split->st.st_size;
}
}
/* now copy the new size in the parity data */
if (is_modified)
*is_modified = 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
if (parity->split_map[s].size != split->size) {
parity->split_map[s].size = split->size;
if (is_modified)
*is_modified = 1;
}
}
return 0;
}
int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode)
{
unsigned s;
handle->level = level;
handle->split_mac = 0;
for (s = 0;s < parity->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
int flags;
/* mark it as initialized */
pathcpy(split->path, sizeof(split->path), parity->split_map[s].path);
split->f = -1;
split->size = parity->split_map[s].size;
++handle->split_mac;
/* open for read */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
/* O_NOATIME: do not change access time */
flags = O_RDONLY | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
split->f = open_noatime(split->path, flags);
if (split->f == -1) {
log_fatal("Error opening parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
}
/* get the stat info */
ret = fstat(split->f, &split->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error advising parity file '%s'. %s.\n", parity->path, strerror(ret));
log_fatal("Error accessing parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/**
* If the parity size is not yet set, set it now.
* This happens when expanding the number of parities.
*/
if (split->size == PARITY_SIZE_INVALID)
split->size = split->st.st_size;
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(split->f, 0, 0, POSIX_FADV_SEQUENTIAL);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(ret));
goto bail;
/* LCOV_EXCL_STOP */
}
}
#endif
}
return 0;
bail:
/* LCOV_EXCL_START */
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
close(split->f);
split->f = -1;
}
return -1;
/* LCOV_EXCL_STOP */
}
int parity_sync(struct snapraid_parity_handle* handle)
{
#if HAVE_FSYNC
unsigned s;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
/* Ensure that data changes are written to disk. */
/* This is required to ensure that parity is more updated than content */
/* in case of a system crash. */
ret = fsync(split->f);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error synching parity file '%s'. %s.\n", split->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
}
#endif
return 0;
bail:
/* LCOV_EXCL_START */
close(parity->f);
parity->f = -1;
parity->valid_size = 0;
return -1;
/* LCOV_EXCL_STOP */
}
int parity_sync(struct snapraid_parity_handle* parity)
int parity_close(struct snapraid_parity_handle* handle)
{
#if HAVE_FSYNC
int ret;
unsigned s;
int f_ret = 0;
/* Ensure that data changes are written to disk. */
/* This is required to ensure that parity is more updated than content */
/* in case of a system crash. */
ret = fsync(parity->f);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error synching parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
ret = close(split->f);
if (ret != 0) {
/* LCOV_EXCL_START */
/* This is a serious error, as it may be the result of a failed write */
/* identified at later time. */
/* In a normal file-system (not NFS) it should never happen */
log_fatal("Error closing parity file '%s'. %s.\n", split->path, strerror(errno));
f_ret = -1;
/* LCOV_EXCL_STOP */
/* continue to close the others */
}
/* reset the descriptor */
split->f = -1;
}
return f_ret;
}
struct snapraid_split_handle* parity_split_find(struct snapraid_parity_handle* handle, data_off_t* offset)
{
unsigned s;
if (*offset < 0)
return 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
if ((uint64_t)*offset < split->size)
return split;
*offset -= split->size;
}
#endif
return 0;
}
int parity_close(struct snapraid_parity_handle* parity)
{
int ret;
ret = close(parity->f);
if (ret != 0) {
/* LCOV_EXCL_START */
/* invalidate for error */
parity->f = -1;
parity->valid_size = 0;
/* This is a serious error, as it may be the result of a failed write */
/* identified at later time. */
/* In a normal file-system (not NFS) it should never happen */
log_fatal("Error closing parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
/* reset the descriptor */
parity->f = -1;
parity->valid_size = 0;
return 0;
}
int parity_write(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size)
int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size)
{
ssize_t write_ret;
data_off_t offset;
struct snapraid_split_handle* split;
offset = pos * (data_off_t)block_size;
write_ret = pwrite(parity->f, block_buffer, block_size, offset);
if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */
split = parity_split_find(handle, &offset);
if (!split) {
/* LCOV_EXCL_START */
if (errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' using write due lack of space.\n", parity->path);
} else {
log_fatal("Error writing file '%s'. %s.\n", parity->path, strerror(errno));
}
log_fatal("Writing parity data outside range at extra offset %" PRIu64 ".\n", offset);
return -1;
/* LCOV_EXCL_STOP */
}
/* adjust the size of the valid data */
if (parity->valid_size < offset + block_size) {
parity->valid_size = offset + block_size;
write_ret = pwrite(split->f, block_buffer, block_size, offset);
if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */
/* LCOV_EXCL_START */
if (errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' using write due lack of space.\n", split->path);
} else {
log_fatal("Error writing file '%s'. %s.\n", split->path, strerror(errno));
}
return -1;
/* LCOV_EXCL_STOP */
}
/* Here doesn't make sense to call posix_fadvise(..., POSIX_FADV_DONTNEED) because */
@ -457,32 +663,35 @@ int parity_write(struct snapraid_parity_handle* parity, block_off_t pos, unsigne
return 0;
}
int parity_read(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out)
int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out)
{
ssize_t read_ret;
data_off_t offset;
unsigned count;
struct snapraid_split_handle* split;
offset = pos * (data_off_t)block_size;
/* check if we are going to read only not initialized data */
if (offset >= parity->valid_size) {
out("Reading missing data from file '%s' at offset %" PRIu64 ".\n", parity->path, offset);
split = parity_split_find(handle, &offset);
if (!split) {
/* LCOV_EXCL_START */
out("Reading parity data outside range at extra offset %" PRIu64 ".\n", offset);
return -1;
/* LCOV_EXCL_STOP */
}
count = 0;
do {
read_ret = pread(parity->f, block_buffer + count, block_size - count, offset + count);
read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count);
if (read_ret < 0) {
/* LCOV_EXCL_START */
out("Error reading file '%s' at offset %" PRIu64 " for size %u. %s.\n", parity->path, offset + count, block_size - count, strerror(errno));
out("Error reading file '%s' at offset %" PRIu64 " for size %u. %s.\n", split->path, offset + count, block_size - count, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
if (read_ret == 0) {
/* LCOV_EXCL_START */
out("Unexpected end of file '%s' at offset %" PRIu64 ". %s.\n", parity->path, offset, strerror(errno));
out("Unexpected end of file '%s' at offset %" PRIu64 ". %s.\n", split->path, offset, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}

46
cmdline/parity.h
View file

@ -23,12 +23,23 @@
/****************************************************************************/
/* parity */
struct snapraid_parity_handle {
struct snapraid_split_handle {
char path[PATH_MAX]; /**< Path of the file. */
unsigned level; /**< Level of the parity. */
int f; /**< Handle of the file. */
int f; /**< Handle of the files. */
struct stat st; /**< Stat info of the opened file. */
data_off_t valid_size; /**< Size of the valid data. */
/**
* Size of the parity split.
* Only the latest not zero size is allowed to grow.
* Note that this value CANNOT be PARITY_SIZE_INVALID.
*/
uint64_t size;
};
struct snapraid_parity_handle {
struct snapraid_split_handle split_map[SPLIT_MAX];
unsigned split_mac; /**< Number of parity splits. */
unsigned level; /**< Level of the parity. */
};
/**
@ -61,52 +72,55 @@ block_off_t parity_used_size(struct snapraid_state* state);
*/
int parity_is_invalid(struct snapraid_state* state);
/**
* If parity has some
*/
int parity_is_invalid(struct snapraid_state* state);
/**
* Report all the files outside the specified parity size.
*/
void parity_overflow(struct snapraid_state* state, data_off_t size);
/**
* Create the parity file.
* \param out_size Return the size of the parity file.
*/
int parity_create(struct snapraid_parity_handle* parity, unsigned level, const char* path, data_off_t* out_size, int mode);
int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode);
/**
* Change the parity size.
* \param out_size Return the size of the parity file. The out_size is set also on error to reflect a partial resize.
*/
int parity_chsize(struct snapraid_parity_handle* parity, data_off_t size, data_off_t* out_size, int skip_fallocate);
int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* parity, int* is_modified, data_off_t size, uint32_t block_size, int skip_fallocate);
/**
* Get the size of opened parity file.
*
* This function never fails as it returns the cached size.
*/
void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size);
/**
* Open an already existing parity file.
*/
int parity_open(struct snapraid_parity_handle* parity, unsigned level, const char* path, int mode);
int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode);
/**
* Flush the parity file in the disk.
*/
int parity_sync(struct snapraid_parity_handle* parity);
int parity_sync(struct snapraid_parity_handle* handle);
/**
* Close the parity file.
*/
int parity_close(struct snapraid_parity_handle* parity);
int parity_close(struct snapraid_parity_handle* handle);
/**
* Read a block from the parity file.
*/
int parity_read(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out);
int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out);
/**
* Write a block in the parity file.
*/
int parity_write(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size);
int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size);
#endif

2
cmdline/scrub.c
View file

@ -854,7 +854,7 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
/* open the file for reading */
for (l = 0; l < state->level; ++l) {
ret = parity_open(&parity_handle[l], l, state->parity[l].path, state->file_mode);
ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to scrub.\n", lev_name(l));

474
cmdline/state.c
View file

@ -138,7 +138,7 @@ const char* lev_raid_name(unsigned mode, unsigned n)
void state_init(struct snapraid_state* state)
{
unsigned l;
unsigned l, s;
memset(&state->opt, 0, sizeof(state->opt));
state->filter_hidden = 0;
@ -149,15 +149,19 @@ void state_init(struct snapraid_state* state)
state->raid_mode = RAID_MODE_CAUCHY;
state->file_mode = MODE_SEQUENTIAL;
for (l = 0; l < LEV_MAX; ++l) {
state->parity[l].path[0] = 0;
state->parity[l].split_mac = 0;
for (s = 0; s < SPLIT_MAX; ++s) {
state->parity[l].split_map[s].path[0] = 0;
state->parity[l].split_map[s].uuid[0] = 0;
state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
state->parity[l].split_map[s].device = 0;
}
state->parity[l].smartctl[0] = 0;
state->parity[l].uuid[0] = 0;
state->parity[l].device = 0;
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
state->parity[l].skip_access = 0;
state->parity[l].tick = 0;
state->parity[l].cached = 0;
state->parity[l].skip_access = 0;
state->parity[l].is_excluded = 0;
}
state->tick_io = 0;
@ -205,7 +209,7 @@ void state_done(struct snapraid_state* state)
static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
{
tommy_node* i;
unsigned l;
unsigned l, s;
/* check for parity level */
if (state->raid_mode == RAID_MODE_VANDERMONDE) {
@ -218,7 +222,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
for (l = 0; l < state->level; ++l) {
if (state->parity[l].path[0] == 0) {
if (state->parity[l].split_mac == 0) {
/* LCOV_EXCL_START */
log_fatal("No '%s' specification in '%s'\n", lev_config_name(l), path);
exit(EXIT_FAILURE);
@ -238,11 +242,13 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
struct snapraid_content* content = i->data;
for (l = 0; l < state->level; ++l) {
if (pathcmp(state->parity[l].path, content->content) == 0) {
/* LCOV_EXCL_START */
log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
for (s = 0; s < state->parity[l].split_mac; ++s) {
if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
/* LCOV_EXCL_START */
log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
@ -302,16 +308,18 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
if (disk->device == state->parity[l].device) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].path);
for (s = 0; s < state->parity[l].split_mac; ++s) {
if (disk->device == state->parity[l].split_map[s].device) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
@ -333,64 +341,64 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
/* check device of parity disks */
if (!state->opt.skip_device && !state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
unsigned j;
for (s = 0; s < state->parity[l].split_mac; ++s) {
unsigned j,t;
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
#ifdef _WIN32
if (state->parity[l].device == 0) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].path);
log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
log_fatal("it's better to change the serial number of the disk.\n");
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (state->parity[l].split_map[s].device == 0) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
log_fatal("it's better to change the serial number of the disk.\n");
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
for (j = l + 1; j < state->level; ++j) {
if (state->parity[l].device == state->parity[j].device) {
if (state->opt.force_device) {
/* note tha we just ignore the issue */
/* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */
if (!state->opt.no_warnings)
log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
} else {
/* LCOV_EXCL_START */
log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].path, state->parity[j].path);
for (j = l + 1; j < state->level; ++j) {
for (t = 0; t < state->parity[j].split_mac; ++t) {
if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
if (state->opt.force_device) {
/* note tha we just ignore the issue */
/* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */
if (!state->opt.no_warnings)
log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
} else {
/* LCOV_EXCL_START */
log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
if (state->pool[0] != 0 && state->pool_device == state->parity[l].device) {
/* LCOV_EXCL_START */
log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].path);
if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
/* LCOV_EXCL_START */
log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
@ -515,7 +523,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
STREAM* f;
unsigned line;
tommy_node* i;
unsigned l;
unsigned l, s;
/* copy the options */
state->opt = *opt;
@ -642,11 +650,13 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
BLOCK_HASH_SIZE = hash_size;
} else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
char device[PATH_MAX];
char* split_map[SPLIT_MAX+1];
unsigned split_mac;
char* slash;
uint64_t dev;
int skip_access;
if (*state->parity[level].path) {
if (state->parity[level].split_mac != 0) {
/* LCOV_EXCL_START */
log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
@ -668,47 +678,61 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* LCOV_EXCL_STOP */
}
pathimport(state->parity[level].path, sizeof(state->parity[level].path), buffer);
split_mac = split(split_map, SPLIT_MAX+1, buffer, ",");
skip_access = 0;
if (!state->opt.skip_parity_access) {
struct stat st;
/* get the device of the directory containing the parity file */
pathimport(device, sizeof(device), buffer);
slash = strrchr(device, '/');
if (slash)
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
skip_access = 1;
log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* use a fake device */
dev = 0;
if (split_mac > SPLIT_MAX) {
/* LCOV_EXCL_START */
log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->parity[level].device = dev;
skip_access = 0;
state->parity[level].split_mac = split_mac;
for (s = 0; s < split_mac; ++s) {
pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
if (!state->opt.skip_parity_access) {
struct stat st;
/* get the device of the directory containing the parity file */
pathimport(device, sizeof(device), split_map[s]);
slash = strrchr(device, '/');
if (slash)
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
skip_access = 1;
log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* use a fake device */
dev = 0;
}
state->parity[level].split_map[s].device = dev;
}
/* store the global parity skip_access */
state->parity[level].skip_access = skip_access;
/* adjust the level */
@ -1201,7 +1225,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
for (l = 0; l < state->level; ++l)
log_tag("%s:%s\n", lev_config_name(l), state->parity[l].path);
for (s = 0; s < state->parity[l].split_mac; ++s)
log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
if (state->pool[0] != 0)
log_tag("pool:%s\n", state->pool);
if (state->share[0] != 0)
@ -1287,7 +1312,7 @@ static void state_map(struct snapraid_state* state)
tommy_node* i;
unsigned uuid_mismatch;
unsigned diskcount;
unsigned l;
unsigned l, s;
/* remove all the mapping without a disk */
/* this happens when a disk is removed from the configuration file */
@ -1410,29 +1435,31 @@ static void state_map(struct snapraid_state* state)
/* check the parity uuid */
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
char uuid[UUID_MAX];
int ret;
for (s = 0; s < state->parity[l].split_mac; ++s) {
char uuid[UUID_MAX];
int ret;
ret = devuuid(state->parity[l].device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore */
continue;
}
/* if the uuid is changed */
if (strcmp(uuid, state->parity[l].uuid) != 0) {
/* if the previous uuid is available */
if (state->parity[l].uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
log_fatal("UUID change for parity '%s' from '%s' to '%s'\n", lev_config_name(l), state->parity[l].uuid, uuid);
ret = devuuid(state->parity[l].split_map[s].device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore */
continue;
}
/* update the uuid */
pathcpy(state->parity[l].uuid, sizeof(state->parity[l].uuid), uuid);
/* if the uuid is changed */
if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
/* if the previous uuid is available */
if (state->parity[l].split_map[s].uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
}
/* write the new state with the new uuid */
state->need_write = 1;
/* update the uuid */
pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
/* write the new state with the new uuid */
state->need_write = 1;
}
}
}
}
@ -1487,7 +1514,7 @@ static void state_map(struct snapraid_state* state)
void state_refresh(struct snapraid_state* state)
{
tommy_node* i;
unsigned l;
unsigned l, s;
/* for all disks */
for (i = state->maplist; i != 0; i = i->next) {
@ -1524,21 +1551,27 @@ void state_refresh(struct snapraid_state* state)
/* for all parities */
for (l = 0; l < state->level; ++l) {
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(state->parity[l].path, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* set the new free blocks */
state->parity[l].total_blocks = total_space / state->block_size;
state->parity[l].free_blocks = free_space / state->block_size;
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
for (s = 0; s < state->parity[l].split_mac; ++s) {
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(state->parity[l].split_map[s].path, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* add the new free blocks */
state->parity[l].total_blocks += total_space / state->block_size;
state->parity[l].free_blocks += free_space / state->block_size;
}
}
/* note what we don't set need_write = 1, because we don't want */
@ -1742,6 +1775,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
* The previous 'm' entry is now deprecated, but supported for importing.
* Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
* The previous 'P' entry is now deprecated, but supported for importing.
*/
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
@ -2559,7 +2594,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
++mapping_max;
} else if (c == 'P') {
/* from SnapRAID 7.0 the 'P' command includes the free space */
char uuid[UUID_MAX];
/* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
char v_uuid[UUID_MAX];
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
@ -2588,7 +2624,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, uuid, sizeof(uuid));
ret = sgetbs(f, v_uuid, sizeof(v_uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
@ -2603,10 +2639,101 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid);
state->parity[v_level].split_mac = 1;
pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
} else if (c == 'Q') {
/* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
uint32_t v_split_mac;
unsigned s;
ret = sgetb32(f, &v_level);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_split_mac);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* auto configure if configuration is missing */
if (state->no_conf && v_level < LEV_MAX && v_level >= state->level)
state->level = v_level + 1;
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
state->parity[v_level].split_mac = v_split_mac;
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
for (s = 0; s < v_split_mac; ++s) {
char v_path[PATH_MAX];
char v_uuid[UUID_MAX];
uint64_t v_size;
ret = sgetbs(f, v_path, sizeof(v_path));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, v_uuid, sizeof(v_uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* if we use this parity entry */
if (v_level < state->level) {
/* we copy the path only if without configuration file */
if (state->no_conf)
pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
state->parity[v_level].split_map[s].size = v_size;
}
}
} else if (c == 'N') {
uint32_t crc_stored;
uint32_t crc_computed;
@ -2716,7 +2843,7 @@ static void* state_write_thread(void* arg)
tommy_node* i;
block_off_t idx;
block_off_t begin;
unsigned l;
unsigned l, s;
count_file = 0;
count_hardlink = 0;
@ -2724,12 +2851,7 @@ static void* state_write_thread(void* arg)
count_dir = 0;
/* write header */
if (BLOCK_HASH_SIZE == HASH_MAX) {
swrite("SNAPCNT2\n\3\0\0", 12, f);
} else {
/* use version 3 only if the hash size is not standard */
swrite("SNAPCNT3\n\3\0\0", 12, f);
}
swrite("SNAPCNT3\n\3\0\0", 12, f);
/* write block size and block max */
sputc('z', f);
@ -2737,11 +2859,9 @@ static void* state_write_thread(void* arg)
sputc('x', f);
sputb32(blockmax, f);
/* write hash size only if not the standard one */
if (BLOCK_HASH_SIZE != HASH_MAX) {
sputc('y', f);
sputb32(BLOCK_HASH_SIZE, f);
}
/* hash size */
sputc('y', f);
sputb32(BLOCK_HASH_SIZE, f);
if (serror(f)) {
/* LCOV_EXCL_START */
@ -2827,11 +2947,16 @@ static void* state_write_thread(void* arg)
/* for each parity */
for (l = 0; l < state->level; ++l) {
sputc('P', f);
sputc('Q', f);
sputb32(l, f);
sputb32(state->parity[l].total_blocks, f);
sputb32(state->parity[l].free_blocks, f);
sputbs(state->parity[l].uuid, f);
sputb32(state->parity[l].split_mac, f);
for (s = 0; s < state->parity[l].split_mac; ++s) {
sputbs(state->parity[l].split_map[s].path, f);
sputbs(state->parity[l].split_map[s].uuid, f);
sputb64(state->parity[l].split_map[s].size, f);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -4403,7 +4528,7 @@ void generate_configuration(const char* path)
{
struct snapraid_state state;
struct snapraid_content* content;
unsigned i;
unsigned l, s;
tommy_node* j;
state_init(&state);
@ -4429,11 +4554,32 @@ void generate_configuration(const char* path)
printf("# Use this hashsize\n");
printf("hashsize %u\n", BLOCK_HASH_SIZE);
printf("\n");
for (i = 0; i < state.level; ++i) {
printf("# Set the correct path for the %s file\n", lev_name(i));
if (state.parity[i].uuid[0])
printf("# The file was in the disk with id '%s'\n", state.parity[i].uuid);
printf("%s ENTER_HERE_THE_PARITY_FILE\n", lev_config_name(i));
for (l = 0; l < state.level; ++l) {
printf("# Set the correct path for the %s files\n", lev_name(l));
printf("# You had %u of them:\n", state.parity[l].split_mac);
for (s = 0; s < state.parity[l].split_mac; ++s) {
printf("# %u:\n", s);
printf("# PATH:");
if (state.parity[l].split_map[s].path[0])
printf("%s", state.parity[l].split_map[s].path);
else
printf("?");
printf("\n");
printf("# SIZE:");
if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
printf("%" PRIu64, state.parity[l].split_map[s].size);
else
printf("?");
printf("\n");
printf("# UUID:");
if (state.parity[l].split_map[s].uuid[0])
printf("%s", state.parity[l].split_map[s].uuid);
else
printf("?");
printf("\n");
printf("#\n");
}
printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
printf("\n");
}
printf("# Add any other content file\n");

54
cmdline/sync.c
View file

@ -1413,7 +1413,6 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
block_off_t used_paritymax;
block_off_t file_paritymax;
data_off_t size;
data_off_t out_size;
int ret;
struct snapraid_parity_handle parity_handle[LEV_MAX];
unsigned unrecoverable_error;
@ -1425,7 +1424,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
blockmax = parity_allocated_size(state);
size = blockmax * (data_off_t)state->block_size;
/* minum size of the parity files we expect */
/* minimum size of the parity files we expect */
used_paritymax = parity_used_size(state);
/* effective size of the parity files */
@ -1444,10 +1443,11 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
for (l = 0; l < state->level; ++l) {
data_off_t out_size;
block_off_t parityblocks;
/* create the file and open for writing */
ret = parity_create(&parity_handle[l], l, state->parity[l].path, &out_size, state->file_mode);
ret = parity_create(&parity_handle[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
@ -1456,11 +1456,12 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
/* number of block in the parity file */
parity_size(&parity_handle[l], &out_size);
parityblocks = out_size / state->block_size;
/* if the file is too small */
if (parityblocks < used_paritymax) {
log_fatal("WARNING! The %s file %s has data only %u blocks instead of %u.\n", lev_name(l), state->parity[l].path, parityblocks, used_paritymax);
log_fatal("WARNING! The %s parity has data only %u blocks instead of %u.\n", lev_name(l), parityblocks, used_paritymax);
}
/* keep the smallest parity number of blocks */
@ -1503,6 +1504,33 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
if (!skip_sync) {
msg_progress("Resizing...\n");
/* now change the size of all parities */
for (l = 0; l < state->level; ++l) {
int is_modified;
/* change the size of the parity file, truncating or extending it */
/* from this point all the DELETED blocks after the end of the parity are invalid */
/* and they are automatically removed when we save the new content file */
ret = parity_chsize(&parity_handle[l], &state->parity[l], &is_modified, size, state->block_size, state->opt.skip_fallocate);
if (ret == -1) {
/* LCOV_EXCL_START */
data_off_t out_size;
parity_size(&parity_handle[l], &out_size);
parity_overflow(state, out_size);
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (is_modified)
state->need_write = 1;
}
/* after resizing parity files, refresh again the free info */
state_refresh(state);
/**
* Save the new state before the sync but after the hashing phase
*
@ -1526,24 +1554,6 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
if (state->need_write)
state_write(state);
/* now change the size of all parities */
for (l = 0; l < state->level; ++l) {
/* change the size of the parity file, truncating or extending it */
/* from this point all the DELETED blocks after the end of the parity are invalid */
/* and they are automatically removed when we save the new content file */
ret = parity_chsize(&parity_handle[l], size, &out_size, state->opt.skip_fallocate);
if (ret == -1) {
/* LCOV_EXCL_START */
parity_overflow(state, out_size);
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* after resizing parity files, refresh again the free info */
state_refresh(state);
msg_progress("Syncing...\n");
/* skip degenerated cases of empty parity, or skipping all */