rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Preliminary support for multiple parity files

Browse source 
This allows to split parity in different partitions.
This commit is contained in:
Andrea Mazzoleni 2016-04-26 21:48:39 +02:00
parent 1ce7b61f29
commit f564e9b91a
11 changed files with 873 additions and 462 deletions
Download patch file Download diff file Expand all files Collapse all files

7
cmdline/check.c
View file

@ -1913,7 +1913,6 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
{
block_off_t blockmax;
data_off_t size;
data_off_t out_size;
int ret;
struct snapraid_parity_handle parity[LEV_MAX];
struct snapraid_parity_handle* parity_ptr[LEV_MAX];
@ -1948,7 +1947,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
}
parity_ptr[l] = &parity[l];
ret = parity_create(parity_ptr[l], l, state->parity[l].path, &out_size, state->file_mode);
ret = parity_create(parity_ptr[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to fix any error.\n", lev_name(l));
@ -1956,7 +1955,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* LCOV_EXCL_STOP */
}
ret = parity_chsize(parity_ptr[l], size, &out_size, state->opt.skip_fallocate);
ret = parity_chsize(parity_ptr[l], &state->parity[l], 0, size, state->block_size, state->opt.skip_fallocate);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
@ -1969,7 +1968,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b
/* it may fail if the file doesn't exist, in this case we continue to check the files */
for (l = 0; l < state->level; ++l) {
parity_ptr[l] = &parity[l];
ret = parity_open(parity_ptr[l], l, state->parity[l].path, state->file_mode);
ret = parity_open(parity_ptr[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
msg_status("No accessible %s file, only files will be checked.\n", lev_name(l));
/* continue anyway */

17
cmdline/device.c
View file

@ -922,19 +922,22 @@ void state_device(struct snapraid_state* state, int operation, tommy_list* filte
/* for all parities */
for (j = 0; j < state->level; ++j) {
devinfo_t* entry;
unsigned s;
if (filterlist_disk != 0 && filter_path(filterlist_disk, 0, lev_config_name(j), 0) != 0)
continue;
entry = calloc_nofail(1, sizeof(devinfo_t));
for (s = 0; s < state->parity[j].split_mac; ++s) {
entry = calloc_nofail(1, sizeof(devinfo_t));
entry->device = state->parity[j].device;
pathcpy(entry->name, sizeof(entry->name), lev_config_name(j));
pathcpy(entry->mount, sizeof(entry->mount), state->parity[j].path);
pathcpy(entry->smartctl, sizeof(entry->smartctl), state->parity[j].smartctl);
pathcut(entry->mount); /* remove the parity file */
entry->device = state->parity[j].split_map[s].device;
pathcpy(entry->name, sizeof(entry->name), lev_config_name(j));
pathcpy(entry->mount, sizeof(entry->mount), state->parity[j].split_map[s].path);
pathcpy(entry->smartctl, sizeof(entry->smartctl), state->parity[j].smartctl);
pathcut(entry->mount); /* remove the parity file */
tommy_list_insert_tail(&high, &entry->node, entry);
tommy_list_insert_tail(&high, &entry->node, entry);
}
}
if (state->opt.fake_device) {

2
cmdline/dry.c
View file

@ -417,7 +417,7 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t
/* open the file for reading */
/* it may fail if the file doesn't exist, in this case we continue to dry the files */
for (l = 0; l < state->level; ++l) {
ret = parity_open(&parity_handle[l], l, state->parity[l].path, state->file_mode);
ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to dry.\n", lev_name(l));

42
cmdline/elem.h
View file

@ -433,19 +433,51 @@ struct snapraid_map {
tommy_node node;
};
/**
* Max number of parity split.
*/
#define SPLIT_MAX 8
/**
* Invalid parity size.
*
* This value is used to identify new parities,
* like when you alter the configuration adding
* a new parity level, creating it with 'fix'.
* Given that 'fix' doesn't write the content file,
* the new size will be written only at the next
* 'sync'.
*/
#define PARITY_SIZE_INVALID -1ULL
/**
* Parity split.
*/
struct snapraid_split {
char path[PATH_MAX]; /**< Path of the parity file. */
char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
/**
* Size of the parity split.
* Only the latest not zero size is allowed to grow.
* If the value is unset, it's PARITY_SIZE_INVALID.
*/
uint64_t size;
uint64_t device; /**< Device identifier of the parity. */
};
/**
* Parity.
*/
struct snapraid_parity {
char path[PATH_MAX]; /**< Path of the parity file. */
char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */
unsigned split_mac; /**< Number of parity splits. */
char smartctl[PATH_MAX]; /**< Custom command for smartctl. Empty means auto. */
uint64_t device; /**< Device identifier of the parity. */
block_off_t total_blocks; /**< Number of total blocks. */
block_off_t free_blocks; /**< Number of free blocks at the last sync. */
int skip_access; /**< If the disk is inaccessible and it should be skipped. */
int is_excluded; /**< If the parity is excluded by disk filter. */
int skip_access; /**< If at least one of the parity disk is inaccessible and it should be skipped. */
uint64_t tick; /**< Usage time. */
uint64_t progress_tick[PROGRESS_MAX]; /**< Last cpu ticks of progress. */
unsigned cached; /**< Number of IO blocks cached. */

681
cmdline/parity.c
View file

@ -150,305 +150,511 @@ void parity_overflow(struct snapraid_state* state, data_off_t size)
}
}
int parity_create(struct snapraid_parity_handle* parity, unsigned level, const char* path, data_off_t* out_size, int mode)
void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size)
{
int ret;
int flags;
unsigned s;
data_off_t size;
parity->level = level;
pathcpy(parity->path, sizeof(parity->path), path);
/* now compute the size */
size = 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
/* opening in sequential mode in Windows */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
flags = O_RDWR | O_CREAT | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
parity->f = open(parity->path, flags, 0600);
if (parity->f == -1) {
/* LCOV_EXCL_START */
log_fatal("Error opening parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
/* check for validity */
if (split->st.st_size < 0)
break;
if ((uint64_t)split->st.st_size < split->size) {
/* this is a size error, so report only up to this size */
/* but don't fail the function */
size += split->st.st_size;
break;
}
/* ignore instead size bigger than expected */
size += split->size;
}
/* get the stat info */
ret = fstat(parity->f, &parity->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
*out_size = size;
}
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(parity->f, 0, 0, POSIX_FADV_SEQUENTIAL);
if (ret != 0) {
int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode)
{
unsigned s;
handle->level = level;
handle->split_mac = 0;
for (s = 0;s < parity->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
int flags;
/* mark it as initialized */
pathcpy(split->path, sizeof(split->path), parity->split_map[s].path);
split->f = -1;
split->size = parity->split_map[s].size;
++handle->split_mac;
/* opening in sequential mode in Windows */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
flags = O_RDWR | O_CREAT | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
split->f = open(split->path, flags, 0600);
if (split->f == -1) {
/* LCOV_EXCL_START */
log_fatal("Error advising parity file '%s'. %s.\n", parity->path, strerror(ret));
log_fatal("Error opening parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
#endif
/* get the size of the existing data */
parity->valid_size = parity->st.st_size;
*out_size = parity->st.st_size;
/* get the stat info */
ret = fstat(split->f, &split->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
return 0;
/**
* If the parity size is not yet set, set it now.
* This happens when expanding the number of parities.
*/
if (split->size == PARITY_SIZE_INVALID)
split->size = split->st.st_size;
bail:
/* LCOV_EXCL_START */
close(parity->f);
parity->f = -1;
parity->valid_size = 0;
return -1;
/* LCOV_EXCL_STOP */
}
int parity_chsize(struct snapraid_parity_handle* parity, data_off_t size, data_off_t* out_size, int skip_fallocate)
{
int ret;
if (parity->st.st_size < size) {
int f_ret;
int f_errno;
const char* call;
#if HAVE_FALLOCATE
call = "fallocate";
if (!skip_fallocate) {
/* allocate real space using the specific Linux fallocate() operation. */
/* If the underline file-system doesn't support it, this operation fails, */
/* instead posix_fallocate() fallbacks to write the whole file. */
ret = fallocate(parity->f, 0, 0, size);
/* fallocate() returns the error number as positive integer, */
/* and in this case it doesn't set errno, just like posix_fallocate() */
/* Checking the glibc code (2.11.1 and 2.14.1) it seems that ENOSYS */
/* may be returned in errno, so we support both the return way */
if (ret > 0) { /* if a positive error is returned, convert it to errno */
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(split->f, 0, 0, POSIX_FADV_SEQUENTIAL);
if (ret != 0) {
/* LCOV_EXCL_START */
errno = ret;
ret = -1;
log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(ret));
goto bail;
/* LCOV_EXCL_STOP */
}
} else {
errno = EOPNOTSUPP;
ret = -1;
}
/* we get EOPNOTSUPP if the operation is not supported, like in ext3/ext2 */
/* or ENOSYS with kernel before 2.6.23 */
if (errno == EOPNOTSUPP || errno == ENOSYS) {
/* fallback using ftruncate() */
call = "ftruncate";
ret = ftruncate(parity->f, size);
}
#else
(void)skip_fallocate; /* avoid the warning */
/* allocate using a sparse file */
call = "ftruncate";
ret = ftruncate(parity->f, size);
#endif
/* save the state of the grow operation */
f_ret = ret;
f_errno = errno;
/* get the stat info */
ret = fstat(parity->f, &parity->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* return the new size */
*out_size = parity->st.st_size;
/* now check the error */
if (f_ret != 0) {
/* LCOV_EXCL_START */
if (f_errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' to size %" PRIu64 " using %s due lack of space.\n", parity->path, size, call);
} else {
log_fatal("Error growing parity file '%s' to size %" PRIu64 " using %s. Do you have enough space? %s.\n", parity->path, size, call, strerror(f_errno));
}
goto bail;
/* LCOV_EXCL_STOP */
}
} else if (parity->st.st_size > size) {
int f_ret;
int f_errno;
/* truncate the parity file */
ret = ftruncate(parity->f, size);
/* save the state of the shrink operation */
f_ret = ret;
f_errno = errno;
/* get the stat info */
ret = fstat(parity->f, &parity->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* return the new size */
*out_size = parity->st.st_size;
/* now check the error */
if (f_ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error truncating parity file '%s' to size %" PRIu64 ". %s.\n", parity->path, size, strerror(f_errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* adjust the valid to the new size */
parity->valid_size = size;
}
return 0;
bail:
/* LCOV_EXCL_START */
parity->valid_size = 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
close(split->f);
split->f = -1;
}
return -1;
/* LCOV_EXCL_STOP */
}
int parity_open(struct snapraid_parity_handle* parity, unsigned level, const char* path, int mode)
static int parity_handle_grow(struct snapraid_split_handle* split, data_off_t size, int skip_fallocate)
{
int ret;
int flags;
parity->level = level;
pathcpy(parity->path, sizeof(parity->path), path);
#if HAVE_FALLOCATE
if (!skip_fallocate) {
/* allocate real space using the specific Linux fallocate() operation. */
/* If the underline file-system doesn't support it, this operation fails, */
/* instead posix_fallocate() fallbacks to write the whole file. */
ret = fallocate(split->f, 0, 0, size);
/* open for read */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
/* O_NOATIME: do not change access time */
flags = O_RDONLY | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
parity->f = open_noatime(parity->path, flags);
if (parity->f == -1) {
log_fatal("Error opening parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* fallocate() returns the error number as positive integer, */
/* and in this case it doesn't set errno, just like posix_fallocate() */
/* Checking the glibc code (2.11.1 and 2.14.1) it seems that ENOSYS */
/* may be returned in errno, so we support both the return way */
if (ret > 0) { /* if a positive error is returned, convert it to errno */
/* LCOV_EXCL_START */
errno = ret;
ret = -1;
/* LCOV_EXCL_STOP */
}
} else {
errno = EOPNOTSUPP;
ret = -1;
}
/* we get EOPNOTSUPP if the operation is not supported, like in ext3/ext2 */
/* or ENOSYS with kernel before 2.6.23 */
if (errno == EOPNOTSUPP || errno == ENOSYS) {
/* fallback using ftruncate() */
ret = ftruncate(split->f, size);
}
#else
(void)skip_fallocate; /* avoid the warning */
/* allocate using a sparse file */
ret = ftruncate(split->f, size);
#endif
return ret;
}
/**
* Get the highest bit set.
*/
uint64_t hbit_u64(uint64_t v)
{
unsigned ilog;
ilog = 0;
while ((v /= 2) != 0)
++ilog;
return 1ULL << ilog;
}
static int parity_handle_fill(struct snapraid_split_handle* split, data_off_t size, uint32_t block_size, int skip_fallocate)
{
data_off_t base;
data_off_t delta;
data_off_t block_mask;
/* mask of bits used by the block size */
block_mask = ((data_off_t)block_size) - 1;
/* present size */
base = split->st.st_size;
/* truncate it to block size multiplier */
/* in case of damage the size may get wrong */
base &= ~block_mask;
/* size we have to increase */
delta = size - base;
/* grow the size one bit at time, like a kind of binary search */
while (delta != 0) {
int ret;
data_off_t run = hbit_u64(delta);
ret = parity_handle_grow(split, base + run, skip_fallocate);
if (ret != 0) {
/* we cannot grow, fallback to the smaller bits */
delta = run - 1;
/* mask out the block size */
delta &= ~block_mask;
} else {
/* mask out this bit */
delta &= ~run;
base += run;
}
}
/* reallocate the expected size */
/* if this fail, is there a problem */
return parity_handle_grow(split, base, skip_fallocate);
}
static int parity_handle_shrink(struct snapraid_split_handle* split, data_off_t size)
{
return ftruncate(split->f, size);
}
static int parity_handle_chsize(struct snapraid_split_handle* split, data_off_t size, uint32_t block_size, int skip_fallocate)
{
int ret;
int f_ret;
int f_errno;
int f_dir;
if (split->st.st_size < size) {
f_ret = parity_handle_fill(split, size, block_size, skip_fallocate);
f_errno = errno;
f_dir = 1;
} else if (split->st.st_size > size) {
f_ret = parity_handle_shrink(split, size);
f_errno = errno;
f_dir = -1;
} else {
f_ret = 0;
f_errno = 0;
f_dir = 0;
}
/* get the stat info */
ret = fstat(parity->f, &parity->st);
ret = fstat(split->f, &split->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing parity file '%s'. %s.\n", parity->path, strerror(errno));
goto bail;
log_fatal("Error accessing parity file '%s'. %s.\n", split->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
/* get the size of the existing data */
parity->valid_size = parity->st.st_size;
/* now check the error */
if (f_ret != 0) {
/* LCOV_EXCL_START */
if (f_dir > 0) {
if (f_errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' to size %" PRIu64 " due lack of space.\n", split->path, size);
} else {
log_fatal("Error growing parity file '%s' to size %" PRIu64 ". Do you have enough space? %s.\n", split->path, size, strerror(f_errno));
}
} else {
log_fatal("Error truncating parity file '%s' to size %" PRIu64 ". %s.\n", split->path, size, strerror(f_errno));
}
return -1;
/* LCOV_EXCL_STOP */
}
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(parity->f, 0, 0, POSIX_FADV_SEQUENTIAL);
return 0;
}
static int parity_split_is_fixed(struct snapraid_parity_handle* handle, unsigned s)
{
/* next one */
++s;
/* the latest one is always growing */
if (s >= handle->split_mac)
return 0;
/* if the next it's 0, this one is growing */
if (handle->split_map[s].size == 0)
return 0;
return 1;
}
int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* parity, int* is_modified, data_off_t size, uint32_t block_size, int skip_fallocate)
{
int ret;
unsigned s;
if (size < 0)
return -1;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int is_fixed = parity_split_is_fixed(handle, s);
data_off_t run;
if (is_fixed) {
if ((uint64_t)size <= split->size) {
is_fixed = 0; /* not fixed anymore */
run = size;
} else {
run = split->size;
}
} else {
run = size;
}
ret = parity_handle_chsize(split, run, block_size, skip_fallocate);
if (ret != 0)
return -1;
if (split->st.st_size > run) {
/* LCOV_EXCL_START */
log_fatal("Unexpected over resizing parity file '%s' to size %" PRIu64 " resulting in size %" PRIu64 ".\n", split->path, run, split->st.st_size);
return -1;
/* LCOV_EXCL_STOP */
} else if (is_fixed && split->st.st_size < run) {
/* LCOV_EXCL_START */
log_fatal("Failed restoring parity file '%s' to size %" PRIu64 " resulting in size %" PRIu64 ".\n", split->path, run, split->st.st_size);
return -1;
/* LCOV_EXCL_STOP */
} else {
size -= split->st.st_size;
split->size = split->st.st_size;
}
}
/* now copy the new size in the parity data */
if (is_modified)
*is_modified = 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
if (parity->split_map[s].size != split->size) {
parity->split_map[s].size = split->size;
if (is_modified)
*is_modified = 1;
}
}
return 0;
}
int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode)
{
unsigned s;
handle->level = level;
handle->split_mac = 0;
for (s = 0;s < parity->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
int flags;
/* mark it as initialized */
pathcpy(split->path, sizeof(split->path), parity->split_map[s].path);
split->f = -1;
split->size = parity->split_map[s].size;
++handle->split_mac;
/* open for read */
/* O_SEQUENTIAL: opening in sequential mode in Windows */
/* O_NOATIME: do not change access time */
flags = O_RDONLY | O_BINARY;
if ((mode & MODE_SEQUENTIAL) != 0)
flags |= O_SEQUENTIAL;
split->f = open_noatime(split->path, flags);
if (split->f == -1) {
log_fatal("Error opening parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
}
/* get the stat info */
ret = fstat(split->f, &split->st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error advising parity file '%s'. %s.\n", parity->path, strerror(ret));
log_fatal("Error accessing parity file '%s'. %s.\n", split->path, strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/**
* If the parity size is not yet set, set it now.
* This happens when expanding the number of parities.
*/
if (split->size == PARITY_SIZE_INVALID)
split->size = split->st.st_size;
#if HAVE_POSIX_FADVISE
if ((mode & MODE_SEQUENTIAL) != 0) {
/* advise sequential access */
ret = posix_fadvise(split->f, 0, 0, POSIX_FADV_SEQUENTIAL);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(ret));
goto bail;
/* LCOV_EXCL_STOP */
}
}
#endif
}
return 0;
bail:
/* LCOV_EXCL_START */
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
close(split->f);
split->f = -1;
}
return -1;
/* LCOV_EXCL_STOP */
}
int parity_sync(struct snapraid_parity_handle* handle)
{
#if HAVE_FSYNC
unsigned s;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
/* Ensure that data changes are written to disk. */
/* This is required to ensure that parity is more updated than content */
/* in case of a system crash. */
ret = fsync(split->f);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error synching parity file '%s'. %s.\n", split->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
}
#endif
return 0;
bail:
/* LCOV_EXCL_START */
close(parity->f);
parity->f = -1;
parity->valid_size = 0;
return -1;
/* LCOV_EXCL_STOP */
}
int parity_sync(struct snapraid_parity_handle* parity)
int parity_close(struct snapraid_parity_handle* handle)
{
#if HAVE_FSYNC
int ret;
unsigned s;
int f_ret = 0;
/* Ensure that data changes are written to disk. */
/* This is required to ensure that parity is more updated than content */
/* in case of a system crash. */
ret = fsync(parity->f);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error synching parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
int ret;
ret = close(split->f);
if (ret != 0) {
/* LCOV_EXCL_START */
/* This is a serious error, as it may be the result of a failed write */
/* identified at later time. */
/* In a normal file-system (not NFS) it should never happen */
log_fatal("Error closing parity file '%s'. %s.\n", split->path, strerror(errno));
f_ret = -1;
/* LCOV_EXCL_STOP */
/* continue to close the others */
}
/* reset the descriptor */
split->f = -1;
}
return f_ret;
}
struct snapraid_split_handle* parity_split_find(struct snapraid_parity_handle* handle, data_off_t* offset)
{
unsigned s;
if (*offset < 0)
return 0;
for (s = 0;s < handle->split_mac;++s) {
struct snapraid_split_handle* split = &handle->split_map[s];
if ((uint64_t)*offset < split->size)
return split;
*offset -= split->size;
}
#endif
return 0;
}
int parity_close(struct snapraid_parity_handle* parity)
{
int ret;
ret = close(parity->f);
if (ret != 0) {
/* LCOV_EXCL_START */
/* invalidate for error */
parity->f = -1;
parity->valid_size = 0;
/* This is a serious error, as it may be the result of a failed write */
/* identified at later time. */
/* In a normal file-system (not NFS) it should never happen */
log_fatal("Error closing parity file '%s'. %s.\n", parity->path, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
/* reset the descriptor */
parity->f = -1;
parity->valid_size = 0;
return 0;
}
int parity_write(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size)
int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size)
{
ssize_t write_ret;
data_off_t offset;
struct snapraid_split_handle* split;
offset = pos * (data_off_t)block_size;
write_ret = pwrite(parity->f, block_buffer, block_size, offset);
if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */
split = parity_split_find(handle, &offset);
if (!split) {
/* LCOV_EXCL_START */
if (errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' using write due lack of space.\n", parity->path);
} else {
log_fatal("Error writing file '%s'. %s.\n", parity->path, strerror(errno));
}
log_fatal("Writing parity data outside range at extra offset %" PRIu64 ".\n", offset);
return -1;
/* LCOV_EXCL_STOP */
}
/* adjust the size of the valid data */
if (parity->valid_size < offset + block_size) {
parity->valid_size = offset + block_size;
write_ret = pwrite(split->f, block_buffer, block_size, offset);
if (write_ret != (ssize_t)block_size) { /* conversion is safe because block_size is always small */
/* LCOV_EXCL_START */
if (errno == ENOSPC) {
log_fatal("Failed to grow parity file '%s' using write due lack of space.\n", split->path);
} else {
log_fatal("Error writing file '%s'. %s.\n", split->path, strerror(errno));
}
return -1;
/* LCOV_EXCL_STOP */
}
/* Here doesn't make sense to call posix_fadvise(..., POSIX_FADV_DONTNEED) because */
@ -457,32 +663,35 @@ int parity_write(struct snapraid_parity_handle* parity, block_off_t pos, unsigne
return 0;
}
int parity_read(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out)
int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out)
{
ssize_t read_ret;
data_off_t offset;
unsigned count;
struct snapraid_split_handle* split;
offset = pos * (data_off_t)block_size;
/* check if we are going to read only not initialized data */
if (offset >= parity->valid_size) {
out("Reading missing data from file '%s' at offset %" PRIu64 ".\n", parity->path, offset);
split = parity_split_find(handle, &offset);
if (!split) {
/* LCOV_EXCL_START */
out("Reading parity data outside range at extra offset %" PRIu64 ".\n", offset);
return -1;
/* LCOV_EXCL_STOP */
}
count = 0;
do {
read_ret = pread(parity->f, block_buffer + count, block_size - count, offset + count);
read_ret = pread(split->f, block_buffer + count, block_size - count, offset + count);
if (read_ret < 0) {
/* LCOV_EXCL_START */
out("Error reading file '%s' at offset %" PRIu64 " for size %u. %s.\n", parity->path, offset + count, block_size - count, strerror(errno));
out("Error reading file '%s' at offset %" PRIu64 " for size %u. %s.\n", split->path, offset + count, block_size - count, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}
if (read_ret == 0) {
/* LCOV_EXCL_START */
out("Unexpected end of file '%s' at offset %" PRIu64 ". %s.\n", parity->path, offset, strerror(errno));
out("Unexpected end of file '%s' at offset %" PRIu64 ". %s.\n", split->path, offset, strerror(errno));
return -1;
/* LCOV_EXCL_STOP */
}

46
cmdline/parity.h
View file

@ -23,12 +23,23 @@
/****************************************************************************/
/* parity */
struct snapraid_parity_handle {
struct snapraid_split_handle {
char path[PATH_MAX]; /**< Path of the file. */
unsigned level; /**< Level of the parity. */
int f; /**< Handle of the file. */
int f; /**< Handle of the files. */
struct stat st; /**< Stat info of the opened file. */
data_off_t valid_size; /**< Size of the valid data. */
/**
* Size of the parity split.
* Only the latest not zero size is allowed to grow.
* Note that this value CANNOT be PARITY_SIZE_INVALID.
*/
uint64_t size;
};
struct snapraid_parity_handle {
struct snapraid_split_handle split_map[SPLIT_MAX];
unsigned split_mac; /**< Number of parity splits. */
unsigned level; /**< Level of the parity. */
};
/**
@ -61,52 +72,55 @@ block_off_t parity_used_size(struct snapraid_state* state);
*/
int parity_is_invalid(struct snapraid_state* state);
/**
* If parity has some
*/
int parity_is_invalid(struct snapraid_state* state);
/**
* Report all the files outside the specified parity size.
*/
void parity_overflow(struct snapraid_state* state, data_off_t size);
/**
* Create the parity file.
* \param out_size Return the size of the parity file.
*/
int parity_create(struct snapraid_parity_handle* parity, unsigned level, const char* path, data_off_t* out_size, int mode);
int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode);
/**
* Change the parity size.
* \param out_size Return the size of the parity file. The out_size is set also on error to reflect a partial resize.
*/
int parity_chsize(struct snapraid_parity_handle* parity, data_off_t size, data_off_t* out_size, int skip_fallocate);
int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* parity, int* is_modified, data_off_t size, uint32_t block_size, int skip_fallocate);
/**
* Get the size of opened parity file.
*
* This function never fails as it returns the cached size.
*/
void parity_size(struct snapraid_parity_handle* handle, data_off_t* out_size);
/**
* Open an already existing parity file.
*/
int parity_open(struct snapraid_parity_handle* parity, unsigned level, const char* path, int mode);
int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_parity* parity, unsigned level, int mode);
/**
* Flush the parity file in the disk.
*/
int parity_sync(struct snapraid_parity_handle* parity);
int parity_sync(struct snapraid_parity_handle* handle);
/**
* Close the parity file.
*/
int parity_close(struct snapraid_parity_handle* parity);
int parity_close(struct snapraid_parity_handle* handle);
/**
* Read a block from the parity file.
*/
int parity_read(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out);
int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size, fptr* out);
/**
* Write a block in the parity file.
*/
int parity_write(struct snapraid_parity_handle* parity, block_off_t pos, unsigned char* block_buffer, unsigned block_size);
int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigned char* block_buffer, unsigned block_size);
#endif

2
cmdline/scrub.c
View file

@ -854,7 +854,7 @@ int state_scrub(struct snapraid_state* state, int plan, int olderthan)
/* open the file for reading */
for (l = 0; l < state->level; ++l) {
ret = parity_open(&parity_handle[l], l, state->parity[l].path, state->file_mode);
ret = parity_open(&parity_handle[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to scrub.\n", lev_name(l));

474
cmdline/state.c
View file

@ -138,7 +138,7 @@ const char* lev_raid_name(unsigned mode, unsigned n)
void state_init(struct snapraid_state* state)
{
unsigned l;
unsigned l, s;
memset(&state->opt, 0, sizeof(state->opt));
state->filter_hidden = 0;
@ -149,15 +149,19 @@ void state_init(struct snapraid_state* state)
state->raid_mode = RAID_MODE_CAUCHY;
state->file_mode = MODE_SEQUENTIAL;
for (l = 0; l < LEV_MAX; ++l) {
state->parity[l].path[0] = 0;
state->parity[l].split_mac = 0;
for (s = 0; s < SPLIT_MAX; ++s) {
state->parity[l].split_map[s].path[0] = 0;
state->parity[l].split_map[s].uuid[0] = 0;
state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
state->parity[l].split_map[s].device = 0;
}
state->parity[l].smartctl[0] = 0;
state->parity[l].uuid[0] = 0;
state->parity[l].device = 0;
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
state->parity[l].skip_access = 0;
state->parity[l].tick = 0;
state->parity[l].cached = 0;
state->parity[l].skip_access = 0;
state->parity[l].is_excluded = 0;
}
state->tick_io = 0;
@ -205,7 +209,7 @@ void state_done(struct snapraid_state* state)
static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
{
tommy_node* i;
unsigned l;
unsigned l, s;
/* check for parity level */
if (state->raid_mode == RAID_MODE_VANDERMONDE) {
@ -218,7 +222,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
for (l = 0; l < state->level; ++l) {
if (state->parity[l].path[0] == 0) {
if (state->parity[l].split_mac == 0) {
/* LCOV_EXCL_START */
log_fatal("No '%s' specification in '%s'\n", lev_config_name(l), path);
exit(EXIT_FAILURE);
@ -238,11 +242,13 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
struct snapraid_content* content = i->data;
for (l = 0; l < state->level; ++l) {
if (pathcmp(state->parity[l].path, content->content) == 0) {
/* LCOV_EXCL_START */
log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
for (s = 0; s < state->parity[l].split_mac; ++s) {
if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
/* LCOV_EXCL_START */
log_fatal("Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
@ -302,16 +308,18 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
if (disk->device == state->parity[l].device) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].path);
for (s = 0; s < state->parity[l].split_mac; ++s) {
if (disk->device == state->parity[l].split_map[s].device) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].split_map[s].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", disk->device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
@ -333,64 +341,64 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
/* check device of parity disks */
if (!state->opt.skip_device && !state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
unsigned j;
for (s = 0; s < state->parity[l].split_mac; ++s) {
unsigned j,t;
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
#ifdef _WIN32
if (state->parity[l].device == 0) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].path);
log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
log_fatal("it's better to change the serial number of the disk.\n");
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (state->parity[l].split_map[s].device == 0) {
/* LCOV_EXCL_START */
log_fatal("Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
log_fatal("This is not necessarily wrong, but for using SnapRAID\n");
log_fatal("it's better to change the serial number of the disk.\n");
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
for (j = l + 1; j < state->level; ++j) {
if (state->parity[l].device == state->parity[j].device) {
if (state->opt.force_device) {
/* note tha we just ignore the issue */
/* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */
if (!state->opt.no_warnings)
log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
} else {
/* LCOV_EXCL_START */
log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].path, state->parity[j].path);
for (j = l + 1; j < state->level; ++j) {
for (t = 0; t < state->parity[j].split_mac; ++t) {
if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
if (state->opt.force_device) {
/* note tha we just ignore the issue */
/* and we DON'T mark the disk to be skipped */
/* because we want to use these disks */
if (!state->opt.no_warnings)
log_fatal("DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
} else {
/* LCOV_EXCL_START */
log_fatal("Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
if (state->pool[0] != 0 && state->pool_device == state->parity[l].device) {
/* LCOV_EXCL_START */
log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].path);
if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
/* LCOV_EXCL_START */
log_fatal("Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
#ifdef _WIN32
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
log_fatal("Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
log_fatal("Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal("to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
@ -515,7 +523,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
STREAM* f;
unsigned line;
tommy_node* i;
unsigned l;
unsigned l, s;
/* copy the options */
state->opt = *opt;
@ -642,11 +650,13 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
BLOCK_HASH_SIZE = hash_size;
} else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
char device[PATH_MAX];
char* split_map[SPLIT_MAX+1];
unsigned split_mac;
char* slash;
uint64_t dev;
int skip_access;
if (*state->parity[level].path) {
if (state->parity[level].split_mac != 0) {
/* LCOV_EXCL_START */
log_fatal("Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
@ -668,47 +678,61 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* LCOV_EXCL_STOP */
}
pathimport(state->parity[level].path, sizeof(state->parity[level].path), buffer);
split_mac = split(split_map, SPLIT_MAX+1, buffer, ",");
skip_access = 0;
if (!state->opt.skip_parity_access) {
struct stat st;
/* get the device of the directory containing the parity file */
pathimport(device, sizeof(device), buffer);
slash = strrchr(device, '/');
if (slash)
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
skip_access = 1;
log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* use a fake device */
dev = 0;
if (split_mac > SPLIT_MAX) {
/* LCOV_EXCL_START */
log_fatal("Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->parity[level].device = dev;
skip_access = 0;
state->parity[level].split_mac = split_mac;
for (s = 0; s < split_mac; ++s) {
pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
if (!state->opt.skip_parity_access) {
struct stat st;
/* get the device of the directory containing the parity file */
pathimport(device, sizeof(device), split_map[s]);
slash = strrchr(device, '/');
if (slash)
*slash = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
skip_access = 1;
log_fatal("DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
} else {
/* LCOV_EXCL_START */
log_fatal("Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal("You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* use a fake device */
dev = 0;
}
state->parity[level].split_map[s].device = dev;
}
/* store the global parity skip_access */
state->parity[level].skip_access = skip_access;
/* adjust the level */
@ -1201,7 +1225,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
for (l = 0; l < state->level; ++l)
log_tag("%s:%s\n", lev_config_name(l), state->parity[l].path);
for (s = 0; s < state->parity[l].split_mac; ++s)
log_tag("%s:%u:%s\n", lev_config_name(l), s, state->parity[l].split_map[s].path);
if (state->pool[0] != 0)
log_tag("pool:%s\n", state->pool);
if (state->share[0] != 0)
@ -1287,7 +1312,7 @@ static void state_map(struct snapraid_state* state)
tommy_node* i;
unsigned uuid_mismatch;
unsigned diskcount;
unsigned l;
unsigned l, s;
/* remove all the mapping without a disk */
/* this happens when a disk is removed from the configuration file */
@ -1410,29 +1435,31 @@ static void state_map(struct snapraid_state* state)
/* check the parity uuid */
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
char uuid[UUID_MAX];
int ret;
for (s = 0; s < state->parity[l].split_mac; ++s) {
char uuid[UUID_MAX];
int ret;
ret = devuuid(state->parity[l].device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore */
continue;
}
/* if the uuid is changed */
if (strcmp(uuid, state->parity[l].uuid) != 0) {
/* if the previous uuid is available */
if (state->parity[l].uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
log_fatal("UUID change for parity '%s' from '%s' to '%s'\n", lev_config_name(l), state->parity[l].uuid, uuid);
ret = devuuid(state->parity[l].split_map[s].device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore */
continue;
}
/* update the uuid */
pathcpy(state->parity[l].uuid, sizeof(state->parity[l].uuid), uuid);
/* if the uuid is changed */
if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
/* if the previous uuid is available */
if (state->parity[l].split_map[s].uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
log_fatal("UUID change for parity '%s[%u]' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
}
/* write the new state with the new uuid */
state->need_write = 1;
/* update the uuid */
pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
/* write the new state with the new uuid */
state->need_write = 1;
}
}
}
}
@ -1487,7 +1514,7 @@ static void state_map(struct snapraid_state* state)
void state_refresh(struct snapraid_state* state)
{
tommy_node* i;
unsigned l;
unsigned l, s;
/* for all disks */
for (i = state->maplist; i != 0; i = i->next) {
@ -1524,21 +1551,27 @@ void state_refresh(struct snapraid_state* state)
/* for all parities */
for (l = 0; l < state->level; ++l) {
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(state->parity[l].path, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* set the new free blocks */
state->parity[l].total_blocks = total_space / state->block_size;
state->parity[l].free_blocks = free_space / state->block_size;
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
for (s = 0; s < state->parity[l].split_mac; ++s) {
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(state->parity[l].split_map[s].path, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal("Error accessing file '%s' to get file-system info. %s.\n", state->parity[l].split_map[s].path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* add the new free blocks */
state->parity[l].total_blocks += total_space / state->block_size;
state->parity[l].free_blocks += free_space / state->block_size;
}
}
/* note what we don't set need_write = 1, because we don't want */
@ -1742,6 +1775,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
* The previous 'm' entry is now deprecated, but supported for importing.
* Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
* The previous 'P' entry is now deprecated, but supported for importing.
*/
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
@ -2559,7 +2594,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
++mapping_max;
} else if (c == 'P') {
/* from SnapRAID 7.0 the 'P' command includes the free space */
char uuid[UUID_MAX];
/* from SnapRAID 11.0 the 'P' command is deprecated by 'Q' */
char v_uuid[UUID_MAX];
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
@ -2588,7 +2624,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, uuid, sizeof(uuid));
ret = sgetbs(f, v_uuid, sizeof(v_uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
@ -2603,10 +2639,101 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid);
state->parity[v_level].split_mac = 1;
pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
} else if (c == 'Q') {
/* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
uint32_t v_split_mac;
unsigned s;
ret = sgetb32(f, &v_level);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_split_mac);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* auto configure if configuration is missing */
if (state->no_conf && v_level < LEV_MAX && v_level >= state->level)
state->level = v_level + 1;
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
state->parity[v_level].split_mac = v_split_mac;
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
for (s = 0; s < v_split_mac; ++s) {
char v_path[PATH_MAX];
char v_uuid[UUID_MAX];
uint64_t v_size;
ret = sgetbs(f, v_path, sizeof(v_path));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, v_uuid, sizeof(v_uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* if we use this parity entry */
if (v_level < state->level) {
/* we copy the path only if without configuration file */
if (state->no_conf)
pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
state->parity[v_level].split_map[s].size = v_size;
}
}
} else if (c == 'N') {
uint32_t crc_stored;
uint32_t crc_computed;
@ -2716,7 +2843,7 @@ static void* state_write_thread(void* arg)
tommy_node* i;
block_off_t idx;
block_off_t begin;
unsigned l;
unsigned l, s;
count_file = 0;
count_hardlink = 0;
@ -2724,12 +2851,7 @@ static void* state_write_thread(void* arg)
count_dir = 0;
/* write header */
if (BLOCK_HASH_SIZE == HASH_MAX) {
swrite("SNAPCNT2\n\3\0\0", 12, f);
} else {
/* use version 3 only if the hash size is not standard */
swrite("SNAPCNT3\n\3\0\0", 12, f);
}
swrite("SNAPCNT3\n\3\0\0", 12, f);
/* write block size and block max */
sputc('z', f);
@ -2737,11 +2859,9 @@ static void* state_write_thread(void* arg)
sputc('x', f);
sputb32(blockmax, f);
/* write hash size only if not the standard one */
if (BLOCK_HASH_SIZE != HASH_MAX) {
sputc('y', f);
sputb32(BLOCK_HASH_SIZE, f);
}
/* hash size */
sputc('y', f);
sputb32(BLOCK_HASH_SIZE, f);
if (serror(f)) {
/* LCOV_EXCL_START */
@ -2827,11 +2947,16 @@ static void* state_write_thread(void* arg)
/* for each parity */
for (l = 0; l < state->level; ++l) {
sputc('P', f);
sputc('Q', f);
sputb32(l, f);
sputb32(state->parity[l].total_blocks, f);
sputb32(state->parity[l].free_blocks, f);
sputbs(state->parity[l].uuid, f);
sputb32(state->parity[l].split_mac, f);
for (s = 0; s < state->parity[l].split_mac; ++s) {
sputbs(state->parity[l].split_map[s].path, f);
sputbs(state->parity[l].split_map[s].uuid, f);
sputb64(state->parity[l].split_map[s].size, f);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -4403,7 +4528,7 @@ void generate_configuration(const char* path)
{
struct snapraid_state state;
struct snapraid_content* content;
unsigned i;
unsigned l, s;
tommy_node* j;
state_init(&state);
@ -4429,11 +4554,32 @@ void generate_configuration(const char* path)
printf("# Use this hashsize\n");
printf("hashsize %u\n", BLOCK_HASH_SIZE);
printf("\n");
for (i = 0; i < state.level; ++i) {
printf("# Set the correct path for the %s file\n", lev_name(i));
if (state.parity[i].uuid[0])
printf("# The file was in the disk with id '%s'\n", state.parity[i].uuid);
printf("%s ENTER_HERE_THE_PARITY_FILE\n", lev_config_name(i));
for (l = 0; l < state.level; ++l) {
printf("# Set the correct path for the %s files\n", lev_name(l));
printf("# You had %u of them:\n", state.parity[l].split_mac);
for (s = 0; s < state.parity[l].split_mac; ++s) {
printf("# %u:\n", s);
printf("# PATH:");
if (state.parity[l].split_map[s].path[0])
printf("%s", state.parity[l].split_map[s].path);
else
printf("?");
printf("\n");
printf("# SIZE:");
if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
printf("%" PRIu64, state.parity[l].split_map[s].size);
else
printf("?");
printf("\n");
printf("# UUID:");
if (state.parity[l].split_map[s].uuid[0])
printf("%s", state.parity[l].split_map[s].uuid);
else
printf("?");
printf("\n");
printf("#\n");
}
printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
printf("\n");
}
printf("# Add any other content file\n");

54
cmdline/sync.c
View file

@ -1413,7 +1413,6 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
block_off_t used_paritymax;
block_off_t file_paritymax;
data_off_t size;
data_off_t out_size;
int ret;
struct snapraid_parity_handle parity_handle[LEV_MAX];
unsigned unrecoverable_error;
@ -1425,7 +1424,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
blockmax = parity_allocated_size(state);
size = blockmax * (data_off_t)state->block_size;
/* minum size of the parity files we expect */
/* minimum size of the parity files we expect */
used_paritymax = parity_used_size(state);
/* effective size of the parity files */
@ -1444,10 +1443,11 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
for (l = 0; l < state->level; ++l) {
data_off_t out_size;
block_off_t parityblocks;
/* create the file and open for writing */
ret = parity_create(&parity_handle[l], l, state->parity[l].path, &out_size, state->file_mode);
ret = parity_create(&parity_handle[l], &state->parity[l], l, state->file_mode);
if (ret == -1) {
/* LCOV_EXCL_START */
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
@ -1456,11 +1456,12 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
/* number of block in the parity file */
parity_size(&parity_handle[l], &out_size);
parityblocks = out_size / state->block_size;
/* if the file is too small */
if (parityblocks < used_paritymax) {
log_fatal("WARNING! The %s file %s has data only %u blocks instead of %u.\n", lev_name(l), state->parity[l].path, parityblocks, used_paritymax);
log_fatal("WARNING! The %s parity has data only %u blocks instead of %u.\n", lev_name(l), parityblocks, used_paritymax);
}
/* keep the smallest parity number of blocks */
@ -1503,6 +1504,33 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
if (!skip_sync) {
msg_progress("Resizing...\n");
/* now change the size of all parities */
for (l = 0; l < state->level; ++l) {
int is_modified;
/* change the size of the parity file, truncating or extending it */
/* from this point all the DELETED blocks after the end of the parity are invalid */
/* and they are automatically removed when we save the new content file */
ret = parity_chsize(&parity_handle[l], &state->parity[l], &is_modified, size, state->block_size, state->opt.skip_fallocate);
if (ret == -1) {
/* LCOV_EXCL_START */
data_off_t out_size;
parity_size(&parity_handle[l], &out_size);
parity_overflow(state, out_size);
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (is_modified)
state->need_write = 1;
}
/* after resizing parity files, refresh again the free info */
state_refresh(state);
/**
* Save the new state before the sync but after the hashing phase
*
@ -1526,24 +1554,6 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
if (state->need_write)
state_write(state);
/* now change the size of all parities */
for (l = 0; l < state->level; ++l) {
/* change the size of the parity file, truncating or extending it */
/* from this point all the DELETED blocks after the end of the parity are invalid */
/* and they are automatically removed when we save the new content file */
ret = parity_chsize(&parity_handle[l], size, &out_size, state->opt.skip_fallocate);
if (ret == -1) {
/* LCOV_EXCL_START */
parity_overflow(state, out_size);
log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* after resizing parity files, refresh again the free info */
state_refresh(state);
msg_progress("Syncing...\n");
/* skip degenerated cases of empty parity, or skipping all */