rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 02:49:02 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions
snapraid/cmdline/state.c
Andrea Mazzoleni 2e71602267 Remove the requirement to snapshot content files 
The new content file format now includes metadata for previous versions
of files. This allows the fix command to access historical file data from
snapshots as needed, ensuring recovery even if the live filesystem has
changed or the file was deleted during a sync.
2026-08-18 23:33:52 +02:00

6031 lines
164 KiB
C
Raw Blame History

// SPDX-License-Identifier: GPL-3.0-or-later
// Copyright (C) 2011 Andrea Mazzoleni
#include "portable.h"
#include "elem.h"
#include "import.h"
#include "search.h"
#include "state.h"
#include "support.h"
#include "parity.h"
#include "stream.h"
#include "handle.h"
#include "io.h"
#include "thermal.h"
#include "raid/raid.h"
#include "raid/cpu.h"
/**
* Configure the multithread support.
*
* Multi thread for write could be either faster or slower, depending
* on the specific conditions. With multithreads it's likely faster
* writing to disk, but you'll need to access multiple times the same data,
* being potentially slower.
*
* Multi thread for verify is instead always generally faster,
* so we enable it if possible.
*/
#if HAVE_THREAD
/* #define HAVE_MT_WRITE 1 */
#define HAVE_MT_VERIFY 1
#endif
const char* lev_name(unsigned l)
{
switch (l) {
case 0 : return "Parity";
case 1 : return "2-Parity";
case 2 : return "3-Parity";
case 3 : return "4-Parity";
case 4 : return "5-Parity";
case 5 : return "6-Parity";
}
return "invalid";
}
const char* lev_config_name(unsigned l)
{
switch (l) {
case 0 : return "parity";
case 1 : return "2-parity";
case 2 : return "3-parity";
case 3 : return "4-parity";
case 4 : return "5-parity";
case 5 : return "6-parity";
}
return "invalid";
}
static int lev_config_scan(const char* s, unsigned* level, unsigned* mode)
{
if (strcmp(s, "parity") == 0 || strcmp(s, "1-parity") == 0) {
*level = 0;
return 0;
}
if (strcmp(s, "q-parity") == 0 || strcmp(s, "2-parity") == 0) {
*level = 1;
return 0;
}
if (strcmp(s, "r-parity") == 0 || strcmp(s, "3-parity") == 0) {
*level = 2;
return 0;
}
if (strcmp(s, "4-parity") == 0) {
*level = 3;
return 0;
}
if (strcmp(s, "5-parity") == 0) {
*level = 4;
return 0;
}
if (strcmp(s, "6-parity") == 0) {
*level = 5;
return 0;
}
if (strcmp(s, "z-parity") == 0) {
*level = 2;
if (mode)
*mode = RAID_MODE_VANDERMONDE;
return 0;
}
return -1;
}
const char* lev_raid_name(unsigned mode, unsigned n)
{
switch (n) {
case 1 : return "par1";
case 2 : return "par2";
case 3 : if (mode == RAID_MODE_CAUCHY)
return "par3";
else
return "parz";
case 4 : return "par4";
case 5 : return "par5";
case 6 : return "par6";
}
return 0;
}
void state_init(struct snapraid_state* state)
{
unsigned l, s, i;
memset(&state->opt, 0, sizeof(state->opt));
state->mapped_device = 0;
state->snapshot = 0;
state->filter_hidden = 0;
state->autosave = 0;
state->need_write = 0;
state->written = 0;
state->checked_read = 0;
state->block_size = 256 * KIBI; /* default 256 KiB */
state->raid_mode = RAID_MODE_CAUCHY;
state->file_mode = ADVISE_DEFAULT;
for (l = 0; l < LEV_MAX; ++l) {
state->parity[l].split_mac = 0;
for (s = 0; s < SPLIT_MAX; ++s) {
state->parity[l].split_map[s].path[0] = 0;
state->parity[l].split_map[s].uuid[0] = 0;
state->parity[l].split_map[s].fstype[0] = 0;
state->parity[l].split_map[s].fslabel[0] = 0;
state->parity[l].split_map[s].size = PARITY_SIZE_INVALID;
state->parity[l].split_map[s].device = 0;
}
state->parity[l].smartctl[0] = 0;
for (i = 0; i < SMART_IGNORE_MAX; ++i)
state->parity[l].smartignore[i] = 0;
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
state->parity[l].skip_access = 0;
state->parity[l].tick = 0;
state->parity[l].cached_blocks = 0;
state->parity[l].is_excluded_by_filter = 0;
}
state->tick_io = 0;
state->tick_misc = 0;
state->tick_sched = 0;
state->tick_raid = 0;
state->tick_hash = 0;
state->tick_last = os_tick();
state->share[0] = 0;
state->pool[0] = 0;
state->pool_device = 0;
state->lockfile[0] = 0;
state->level = 1; /* default is the lowest protection */
state->no_conf = 0;
for (i = 0; i < SMART_IGNORE_MAX; ++i)
state->smartignore[i] = 0;
state->rehash_blocks = 0;
state->bad_blocks = 0;
state->unsynced_blocks = 0;
state->unscrubbed_blocks = 0;
state->thermal_stop_gathering = 0;
state->thermal_ambient_temperature = 0;
state->thermal_highest_temperature = 0;
state->thermal_first = 0;
state->thermal_latest = 0;
state->thermal_temperature_limit = 0;
state->thermal_cooldown_time = 0;
tommy_list_init(&state->disklist);
tommy_list_init(&state->extralist);
tommy_list_init(&state->maplist);
tommy_list_init(&state->contentlist);
tommy_list_init(&state->filterlist);
tommy_list_init(&state->importlist);
tommy_list_init(&state->thermallist);
tommy_hashdyn_init(&state->importset);
tommy_hashdyn_init(&state->previmportset);
tommy_hashdyn_init(&state->searchset);
tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
tommy_list_init(&state->bucketlist);
}
void state_done(struct snapraid_state* state)
{
tommy_list_foreach(&state->disklist, (tommy_foreach_func*)disk_free);
tommy_list_foreach(&state->extralist, (tommy_foreach_func*)extra_free);
tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free);
tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
tommy_list_foreach(&state->thermallist, (tommy_foreach_func*)thermal_free);
tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
tommy_hashdyn_done(&state->importset);
tommy_hashdyn_done(&state->previmportset);
tommy_hashdyn_done(&state->searchset);
tommy_arrayblkof_done(&state->infoarr);
tommy_list_foreach(&state->bucketlist, (tommy_foreach_func*)bucket_free);
}
/**
* Check the configuration.
*/
static void state_config_check(struct snapraid_state* state, const char* path, tommy_list* filterlist_disk)
{
tommy_node* i;
unsigned l, s;
/* check for parity level */
if (state->raid_mode == RAID_MODE_VANDERMONDE) {
if (state->level > 3) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Using z-parity limits you to a maximum of 3 parities.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STIO */
}
}
for (l = 0; l < state->level; ++l) {
if (state->parity[l].split_mac == 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Missing '%s' specification in '%s'\n", lev_config_name(l), path);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
if (tommy_list_empty(&state->contentlist)) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Missing 'content' specification in '%s'\n", path);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check for equal paths */
for (i = state->contentlist; i != 0; i = i->next) {
struct snapraid_content* content = i->data;
for (l = 0; l < state->level; ++l) {
for (s = 0; s < state->parity[l].split_mac; ++s) {
if (pathcmp(state->parity[l].split_map[s].path, content->content) == 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Same path used for '%s' and 'content' as '%s'\n", lev_config_name(l), content->content);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
/* check device of data disks */
if (!state->opt.skip_device && !state->opt.skip_disk_access) {
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
/* skip data disks that are not accessible */
if (disk->skip_access)
continue;
#ifdef _WIN32
if (disk->dir_device == 0) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Disk '%s' has a zero serial number.\n", disk->dir);
log_fatal(ESOFT, "This is not necessarily wrong, but for using SnapRAID\n");
log_fatal(ESOFT, "it's better to change the serial number of the disk.\n");
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
for (j = i->next; j != 0; j = j->next) {
struct snapraid_disk* other = j->data;
if (disk->mount_device == other->mount_device) {
if (state->opt.force_device) {
/*
* Note that we just ignore the issue
* and we DON'T mark the disk to be skipped
* because we want to use these disks
*/
if (!state->opt.no_warnings)
log_fatal(EUSER, "DANGER! Ignoring that disks '%s' and '%s' are on the same device\n", disk->name, other->name);
} else {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Disks '%s' and '%s' are on the same device.\n", disk->mount_point, other->mount_point);
#ifdef _WIN32
log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", disk->dir_device);
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal(ESOFT, "to change one of the disk serial.\n");
#endif
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal(ESOFT, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
for (s = 0; s < state->parity[l].split_mac; ++s) {
if (disk->mount_device == state->parity[l].split_map[s].device) {
if (state->opt.force_device) {
/*
* Note that we just ignore the issue
* and we DON'T mark the disk to be skipped
* because we want to use these disks
*/
if (!state->opt.no_warnings)
log_fatal(EUSER, "DANGER! Ignoring that disks '%s' and %s '%s' are on the same device\n", disk->mount_point, lev_name(l), state->parity[l].split_map[s].path);
} else {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Disk '%s' and %s '%s' are on the same device.\n", disk->mount_point, lev_name(l), state->parity[l].split_map[s].path);
#ifdef _WIN32
log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", disk->dir_device);
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal(ESOFT, "to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
}
if (state->pool[0] != 0 && disk->mount_device == state->pool_device) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Disk '%s' and pool '%s' are on the same device.\n", disk->mount_point, state->pool);
#ifdef _WIN32
log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", disk->dir_device);
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal(ESOFT, "to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
/* check device of parity disks */
if (!state->opt.skip_device && !state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
for (s = 0; s < state->parity[l].split_mac; ++s) {
unsigned j, t;
/* skip parity disks that are not accessible */
if (state->parity[l].skip_access)
continue;
#ifdef _WIN32
if (state->parity[l].split_map[s].device == 0) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Disk '%s' has a zero serial number.\n", state->parity[l].split_map[s].path);
log_fatal(ESOFT, "This is not necessarily wrong, but for using SnapRAID\n");
log_fatal(ESOFT, "it's better to change the serial number of the disk.\n");
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
for (j = l + 1; j < state->level; ++j) {
for (t = 0; t < state->parity[j].split_mac; ++t) {
if (state->parity[l].split_map[s].device == state->parity[j].split_map[t].device) {
if (state->opt.force_device) {
/*
* Note that we just ignore the issue
* and we DON'T mark the disk to be skipped
* because we want to use these disks
*/
if (!state->opt.no_warnings)
log_fatal(EUSER, "DANGER! Skipping parities '%s' and '%s' on the same device\n", lev_config_name(l), lev_config_name(j));
} else {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Parity '%s' and '%s' are on the same device.\n", state->parity[l].split_map[s].path, state->parity[j].split_map[t].path);
#ifdef _WIN32
log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", state->parity[l].split_map[s].device);
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal(ESOFT, "to change one of the disk serial.\n");
#endif
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal(ESOFT, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
if (state->pool[0] != 0 && state->pool_device == state->parity[l].split_map[s].device) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Pool '%s' and parity '%s' are on the same device.\n", state->pool, state->parity[l].split_map[s].path);
#ifdef _WIN32
log_fatal(ESOFT, "Both have the serial number '%" PRIx64 "'.\n", state->pool_device);
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
log_fatal(ESOFT, "to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
/* check device of pool disk */
#ifdef _WIN32
if (!state->opt.skip_device) {
if (state->pool[0] != 0 && state->pool_device == 0) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Disk '%s' has a zero serial number.\n", state->pool);
log_fatal(ESOFT, "This is not necessarily wrong, but for using SnapRAID\n");
log_fatal(ESOFT, "it's better to change the serial number of the disk.\n");
log_fatal(ESOFT, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
#endif
/* count the content files */
if (!state->opt.skip_device && !state->opt.skip_content_access) {
unsigned content_count;
content_count = 0;
for (i = state->contentlist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_content* content = i->data;
/* check if there are others in the same disk */
for (j = i->next; j != 0; j = j->next) {
struct snapraid_content* other = j->data;
if (content->device == other->device) {
log_error(ESOFT, "WARNING! Content files on the same disk: '%s' and '%s'.\n", content->content, other->content);
break;
}
}
if (j != 0) {
/* skip it */
continue;
}
++content_count;
}
if (content_count < state->level + 1) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You must have at least %d 'content' files in different disks.\n", state->level + 1);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* check for speed */
#ifdef CONFIG_X86
if (!raid_cpu_has_ssse3())
#endif
if (state->raid_mode == RAID_MODE_CAUCHY && !state->opt.no_warnings) {
if (state->level == 3) {
log_info(EUSER, "WARNING! Your CPU doesn't have a fast implementation for triple parity.\n");
log_info(EUSER, "WARNING! It's recommended to switch to 'z-parity' instead than '3-parity'.\n");
} else if (state->level > 3) {
log_info(EUSER, "WARNING! Your CPU doesn't have a fast implementation beyond triple parity.\n");
log_info(EUSER, "WARNING! It's recommended to reduce the parity levels to triple parity.\n");
}
}
/* ensure that specified filter disks are valid ones */
for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
tommy_node* j;
struct snapraid_filter* filter = i->data;
for (j = state->disklist; j != 0; j = j->next) {
struct snapraid_disk* disk = j->data;
if (wnmatch(filter->pattern, disk->name) == 0)
break;
}
if (j == 0) {
for (j = state->extralist; j != 0; j = j->next) {
struct snapraid_extra* extra = j->data;
if (wnmatch(filter->pattern, extra->name) == 0)
break;
}
}
if (j == 0) {
/* check matching with parity disks */
for (l = 0; l < state->level; ++l)
if (wnmatch(filter->pattern, lev_config_name(l)) == 0)
break;
if (l == state->level) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Option -d, --filter-disk %s doesn't match any data or parity or extra disk.\n", filter->pattern);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
/**
* Validate the smartctl command.
*
* It must contains only one %s string, and not other % chars.
*/
static int validate_smartctl(const char* custom)
{
const char* s = custom;
int arg = 0;
while (*s) {
if (s[0] == '%' && s[1] == 's') {
if (arg) {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
arg = 1;
} else if (s[0] == '%') {
/* LCOV_EXCL_START */
return -1;
/* LCOV_EXCL_STOP */
}
++s;
}
return 0;
}
void state_config(struct snapraid_state* state, const char* path, const char* command, struct snapraid_option* opt, tommy_list* filterlist_disk)
{
STREAM* f;
unsigned line;
tommy_node* i;
unsigned l, s;
/* copy the options */
state->opt = *opt;
/* if unset, sort by physical order */
if (!state->opt.force_order)
state->opt.force_order = SORT_PHYSICAL;
/* adjust file mode */
if (state->opt.file_mode != ADVISE_DEFAULT) {
state->file_mode = state->opt.file_mode;
} else {
/* default mode, if nothing is specified */
state->file_mode = ADVISE_DISCARD;
}
/* store current command */
state->command = command;
log_tag("conf:file:%s\n", esc_tag(path));
f = sopen_read(path, 0);
if (!f) {
/* LCOV_EXCL_START */
if (errno == ENOENT) {
log_fatal(errno, "No configuration file found at '%s'\n", path);
} else if (errno == EACCES) {
log_fatal(errno, "You do not have rights to access the configuration file '%s'\n", path);
} else {
log_fatal(errno, "Error opening the configuration file '%s'. %s.\n", path, strerror(errno));
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
line = 1;
while (1) {
char tag[PATH_MAX];
char buffer[PATH_MAX];
int ret;
int c;
unsigned level;
/* skip initial spaces */
sgetspace(f);
/* read the command */
ret = sgettok(f, tag, sizeof(tag));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Error reading the configuration file '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* skip spaces after the command */
sgetspace(f);
if (strcmp(tag, "blocksize") == 0
/* block_size is the old format of the option */
|| strcmp(tag, "block_size") == 0) {
ret = sgetu32(f, &state->block_size);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'blocksize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (state->block_size < 1) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Too small 'blocksize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (state->block_size > 16 * KIBI) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Too big 'blocksize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check if it's a power of 2 */
if ((state->block_size & (state->block_size - 1)) != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Not power of 2 'blocksize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->block_size *= KIBI;
} else if (strcmp(tag, "hashsize") == 0
|| strcmp(tag, "hash_size") == 0 /* v11.0 used incorrectly this one, kept now for backward compatibility */
) {
uint32_t hash_size;
ret = sgetu32(f, &hash_size);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (hash_size < 2) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Too small 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (hash_size > HASH_MAX) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Too big 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check if it's a power of 2 */
if ((hash_size & (hash_size - 1)) != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
BLOCK_HASH_SIZE = hash_size;
} else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
char device[PATH_MAX];
char* split_map[SPLIT_MAX + 1];
unsigned split_mac;
uint64_t dev;
int skip_access;
if (state->parity[level].split_mac != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Multiple '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
split_mac = strsplit(split_map, SPLIT_MAX + 1, buffer, ",");
if (split_mac > SPLIT_MAX) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Too many files in '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
skip_access = 0;
state->parity[level].split_mac = split_mac;
for (s = 0; s < split_mac; ++s) {
char uuid[UUID_MAX];
pathimport(state->parity[level].split_map[s].path, sizeof(state->parity[level].split_map[s].path), split_map[s]);
if (!state->opt.skip_parity_access) {
struct stat st;
/* get the device of the directory containing the parity file */
pathimport(device, sizeof(device), split_map[s]);
/* use only the dir, as the parity file may not exist yet */
pathcut(device);
if (stat(device, &st) == 0) {
dev = st.st_dev;
/* read the uuid, if unsupported use an empty one */
if (devuuid(dev, device, uuid, sizeof(uuid)) != 0) {
*uuid = 0;
}
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
*uuid = 0;
skip_access = 1;
log_fatal(errno, "DANGER! Skipping inaccessible parity disk '%s'...\n", tag);
} else {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal(errno, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* use a fake device */
dev = 0;
*uuid = 0;
}
state->parity[level].split_map[s].device = dev;
pathcpy(state->parity[level].split_map[s].uuid, sizeof(state->parity[level].split_map[s].uuid), uuid);
}
/* store the global parity skip_access */
state->parity[level].skip_access = skip_access;
/* adjust the level */
if (state->level < level + 1)
state->level = level + 1;
} else if (strcmp(tag, "share") == 0) {
if (*state->share) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Multiple 'share' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'share' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'share' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
pathimport(state->share, sizeof(state->share), buffer);
} else if (strcmp(tag, "pool") == 0) {
struct stat st;
if (*state->pool) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Multiple 'pool' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'pool' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'pool' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
pathimport(state->pool, sizeof(state->pool), buffer);
/* get the device of the directory containing the pool tree */
if (stat(buffer, &st) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing 'pool' dir '%s' specification in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->pool_device = st.st_dev;
} else if (strcmp(tag, "content") == 0) {
struct snapraid_content* content;
char device[PATH_MAX];
char* slash;
struct stat st;
uint64_t dev;
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'content' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (pathcmp(buffer, "/dev/null") == 0 || pathcmp(buffer, "NUL") == 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "You cannot use the null device as 'content' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'content' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check if the content file is already specified */
for (i = state->contentlist; i != 0; i = i->next) {
content = i->data;
if (pathcmp(content->content, buffer) == 0)
break;
}
if (i) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Duplicate 'content' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* get the device of the directory containing the content file */
pathimport(device, sizeof(device), buffer);
slash = strrchr(device, '/');
if (slash)
slash[1] = 0;
else
pathcpy(device, sizeof(device), ".");
if (stat(device, &st) == 0) {
dev = st.st_dev;
} else {
if (state->opt.skip_content_access) {
/* use a fake device */
dev = 0;
log_fatal(errno, "WARNING! Skipping inaccessible content file '%s'...\n", buffer);
} else {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* set the lock file at the first accessible content file */
if (state->lockfile[0] == 0 && dev != 0) {
pathcpy(state->lockfile, sizeof(state->lockfile), buffer);
pathcat(state->lockfile, sizeof(state->lockfile), ".lock");
}
content = content_alloc(buffer, dev);
tommy_list_insert_tail(&state->contentlist, &content->node, content);
} else if (strcmp(tag, "data") == 0 || strcmp(tag, "disk") == 0) {
/* "disk" is the deprecated name up to SnapRAID 9.x */
char dir[PATH_MAX];
char device[PATH_MAX];
char uuid[UUID_MAX];
struct snapraid_disk* disk;
uint64_t dev;
int skip_access;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'data' name specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'data' name specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
sgetspace(f);
ret = sgetlasttok(f, dir, sizeof(dir));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'data' dir specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*dir) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'data' dir specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* get the device of the dir */
pathimport(device, sizeof(device), dir);
/* check if the disk name already exists */
for (i = state->disklist; i != 0; i = i->next) {
disk = i->data;
if (strcmp(disk->name, buffer) == 0)
break;
}
if (i) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Duplicate 'data' name '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* if the disk has to be present */
skip_access = 0;
if (!state->opt.skip_disk_access) {
struct stat st;
if (stat(device, &st) == 0) {
dev = st.st_dev;
/* read the uuid, if unsupported use an empty one */
if (devuuid(dev, device, uuid, sizeof(uuid)) != 0) {
*uuid = 0;
}
/* fake a different UUID when testing */
if (state->opt.fake_uuid) {
snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
--state->opt.fake_uuid;
}
} else {
/* if the disk can be skipped */
if (state->opt.force_device) {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
*uuid = 0;
skip_access = 1;
log_fatal(errno, "DANGER! Skipping inaccessible data disk '%s'...\n", buffer);
} else {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
/* in "fix" we allow to continue anyway */
if (strcmp(state->command, "fix") == 0) {
log_fatal(errno, "You can '%s' anyway, using 'snapraid --force-device %s'.\n", state->command, state->command);
}
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else {
/* use a fake device */
dev = 0;
*uuid = 0;
}
disk = disk_alloc(buffer, dir, dev, uuid, skip_access);
tommy_list_insert_tail(&state->disklist, &disk->node, disk);
} else if (strcmp(tag, "extra") == 0) {
char dir[PATH_MAX];
char device[PATH_MAX];
char uuid[UUID_MAX];
struct snapraid_extra* extra;
uint64_t dev;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'extra' name specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'extra' name specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
sgetspace(f);
ret = sgetlasttok(f, dir, sizeof(dir));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'extra' dir specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*dir) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'extra' dir specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* get the device of the dir */
pathimport(device, sizeof(device), dir);
/* check if the disk name already exists */
for (i = state->extralist; i != 0; i = i->next) {
extra = i->data;
if (strcmp(extra->name, buffer) == 0)
break;
}
if (i) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Duplicate 'extra' name '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* if the disk has to be present */
if (!state->opt.skip_disk_access) {
struct stat st;
if (stat(device, &st) == 0) {
dev = st.st_dev;
/* read the uuid, if unsupported use an empty one */
if (devuuid(dev, device, uuid, sizeof(uuid)) != 0) {
*uuid = 0;
}
/* fake a different UUID when testing */
if (state->opt.fake_uuid) {
snprintf(uuid, sizeof(uuid), "fake-uuid-%d", state->opt.fake_uuid);
--state->opt.fake_uuid;
}
} else {
/* use a fake device, and mark the disk to be skipped */
dev = 0;
*uuid = 0;
}
} else {
/* use a fake device */
dev = 0;
*uuid = 0;
}
extra = extra_alloc(buffer, dir, dev, uuid);
tommy_list_insert_tail(&state->extralist, &extra->node, extra);
} else if (strcmp(tag, "smartctl") == 0) {
char custom[PATH_MAX];
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartctl' name in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'smartctl' name in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
sgetspace(f);
ret = sgetlasttok(f, custom, sizeof(custom));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartctl' option in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*custom) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'smartctl' option in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (validate_smartctl(custom) != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartctl' option in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* search for parity */
if (lev_config_scan(buffer, &level, 0) == 0) {
if (state->parity[level].smartctl[0] != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Duplicate parity 'smartctl' '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
pathcpy(state->parity[level].smartctl, sizeof(state->parity[level].smartctl), custom);
} else {
/* search the disk */
struct snapraid_disk* found_disk = 0;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (strcmp(disk->name, buffer) == 0) {
found_disk = disk;
break;
}
}
struct snapraid_extra* found_extra = 0;
for (i = state->extralist; i != 0; i = i->next) {
struct snapraid_extra* extra = i->data;
if (strcmp(extra->name, buffer) == 0) {
found_extra = extra;
break;
}
}
if (found_disk) {
if (found_disk->smartctl[0] != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Duplicate 'smartctl' name '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
pathcpy(found_disk->smartctl, sizeof(found_disk->smartctl), custom);
} else if (found_extra) {
if (found_extra->smartctl[0] != 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Duplicate 'smartctl' name '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
pathcpy(found_extra->smartctl, sizeof(found_extra->smartctl), custom);
} else {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Unknown 'smartctl' name '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
} else if (strcmp(tag, "smartignore") == 0) {
int* smart;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartignore' name in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'smartignore' name in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
sgetspace(f);
if (strcmp(buffer, "*") == 0) {
smart = state->smartignore;
} else if (lev_config_scan(buffer, &level, 0) == 0) { /* search for parity */
smart = state->parity[level].smartignore;
} else {
/* search the disk */
struct snapraid_disk* found_disk = 0;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (strcmp(disk->name, buffer) == 0) {
found_disk = disk;
break;
}
}
struct snapraid_extra* found_extra = 0;
for (i = state->extralist; i != 0; i = i->next) {
struct snapraid_extra* extra = i->data;
if (strcmp(extra->name, buffer) == 0) {
found_extra = extra;
break;
}
}
if (found_disk) {
smart = found_disk->smartignore;
} else if (found_extra) {
smart = found_extra->smartignore;
} else {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Unknown 'smartignore' name '%s' at line %u\n", buffer, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
int si = 0;
while (1) {
char* e;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer)
break;
if (si == SMART_IGNORE_MAX) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Too many 'smartignore' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
smart[si] = strtoul(buffer, &e, 0);
if (!e || *e) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u. It should be a number.\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (smart[si] < 1 || smart[si] >= 256) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u. It should be between 1 and 255.\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
++si;
sgetspace(f);
}
if (si == 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'smartignore' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else if (strcmp(tag, "temp_limit") == 0) {
unsigned temp;
ret = sgetu32(f, &temp);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'temp_limit' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (temp < 40 || temp > 70) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'temp_limit' temperature specification in '%s' at line %u. It must be between 40 and 70\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->thermal_temperature_limit = temp;
} else if (strcmp(tag, "temp_sleep") == 0) {
unsigned time;
ret = sgetu32(f, &time);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'temp_sleep' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (time < 3 || time > 120) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'temp_sleep' temperature specification in '%s' at line %u. It must be between 3 and 120\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->thermal_cooldown_time = time * 60;
} else if (strcmp(tag, "nohidden") == 0) {
state->filter_hidden = 1;
} else if (strcmp(tag, "snapshot") == 0) {
state->snapshot = 1;
} else if (strcmp(tag, "exclude") == 0) {
struct snapraid_filter* filter;
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'exclude' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'exclude' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
filter = filter_alloc_file(-1, "", buffer);
if (!filter) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'exclude' specification '%s' in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
} else if (strcmp(tag, "include") == 0) {
struct snapraid_filter* filter;
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'include' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'include' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
filter = filter_alloc_file(1, "", buffer);
if (!filter) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'include' specification '%s' in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
} else if (strcmp(tag, "autosave") == 0) {
char* e;
ret = sgetlasttok(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'autosave' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!*buffer) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Empty 'autosave' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
state->autosave = strtoul(buffer, &e, 0);
if (!e || *e) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid 'autosave' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* convert to GB */
state->autosave *= GIGA;
} else if (tag[0] == 0) {
/* allow empty lines */
} else if (tag[0] == '#') {
ret = sgetline(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid comment in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Invalid command '%s' in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* skip final spaces */
sgetspace(f);
/* next line */
c = sgeteol(f);
if (c == EOF) {
break;
}
if (c != '\n') {
/* LCOV_EXCL_START */
log_fatal(EUSER, "Extra data in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
++line;
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error reading the configuration file '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
sclose(f);
state_config_check(state, path, filterlist_disk);
/* select the default hash */
if (state->opt.force_murmur3) {
state->besthash = HASH_MURMUR3;
} else if (state->opt.force_spooky2) {
state->besthash = HASH_SPOOKY2;
} else {
state->besthash = membesthash();
}
/* by default use the best hash */
state->hash = state->besthash;
/* by default use a random hash seed */
if (randomize(state->hashseed, HASH_MAX) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Failed to retrieve random values.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* no previous hash by default */
state->prevhash = HASH_UNDEFINED;
/* intentionally not set the prevhashseed, if used valgrind will warn about it */
log_tag("blocksize:%u\n", state->block_size);
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
log_tag("data:%s:%s:%s\n", esc_tag(disk->name), esc_tag(disk->mount_point), disk->uuid);
}
for (i = state->extralist; i != 0; i = i->next) {
struct snapraid_extra* extra = i->data;
log_tag("extra:%s:%s:%s\n", esc_tag(extra->name), esc_tag(extra->dir), extra->uuid);
}
log_tag("mode:%s\n", lev_raid_name(state->raid_mode, state->level));
for (l = 0; l < state->level; ++l) {
log_tag("%s:%s:%s\n", lev_config_name(l), esc_tag(state->parity[l].split_map[0].path), state->parity[l].split_map[0].uuid);
for (s = 1; s < state->parity[l].split_mac; ++s) {
log_tag("%s/%u:%s:%s\n", lev_config_name(l), s, esc_tag(state->parity[l].split_map[s].path), state->parity[l].split_map[s].uuid);
}
}
if (state->pool[0] != 0)
log_tag("pool:%s\n", esc_tag(state->pool));
if (state->share[0] != 0)
log_tag("share:%s\n", esc_tag(state->share));
if (state->autosave != 0)
log_tag("autosave:%" PRIu64 "\n", state->autosave);
for (i = tommy_list_head(&state->filterlist); i != 0; i = i->next) {
char out[PATH_MAX];
struct snapraid_filter* filter = i->data;
log_tag("filter:%s\n", esc_tag(filter_type(filter, out, sizeof(out))));
}
if (state->filter_hidden)
log_tag("filter:nohidden:\n");
log_flush();
}
/**
* Find a disk by name.
*/
static struct snapraid_disk* find_disk_by_name(struct snapraid_state* state, const char* name)
{
tommy_node* i;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (strcmp(disk->name, name) == 0)
return disk;
}
if (state->no_conf) {
/* without a configuration file, add disks automatically */
struct snapraid_disk* disk;
disk = disk_alloc(name, "DUMMY/", -1, "", 0);
tommy_list_insert_tail(&state->disklist, &disk->node, disk);
return disk;
}
return 0;
}
/**
* Find a disk by UUID.
*/
static struct snapraid_disk* find_disk_by_uuid(struct snapraid_state* state, const char* uuid)
{
tommy_node* i;
struct snapraid_disk* found = 0;
/*
* Special test case to find the first matching UUID
* when testing UUID are all equal or not supported
* and we should handle this case specifically
*/
if (state->opt.match_first_uuid)
return state->disklist->data;
/* LCOV_EXCL_START */
/* never find an empty uuid */
if (!*uuid)
return 0;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (strcmp(disk->uuid, uuid) == 0) {
/* never match duplicate UUID */
if (found)
return 0;
found = disk;
}
}
return found;
/* LCOV_EXCL_STOP */
}
/**
* Update the disk mapping if required.
*/
static void state_map(struct snapraid_state* state)
{
unsigned hole;
tommy_node* i;
unsigned uuid_mismatch;
unsigned diskcount;
unsigned l, s;
/*
* Remove all the mapping without a disk
* this happens when a disk is removed from the configuration file
* From SnapRAID 4.0 mappings are automatically removed if a disk is not used
* when saving the content file, but we keep this code to import older content files.
*/
for (i = state->maplist; i != 0; ) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
disk = find_disk_by_name(state, map->name);
/* go to the next mapping before removing */
i = i->next;
if (disk == 0) {
/* disk not found, remove the mapping */
tommy_list_remove_existing(&state->maplist, &map->node);
map_free(map);
}
}
/*
* Maps each unmapped disk present in the configuration file in the first available hole
* this happens when you add disks for the first time in the configuration file
*/
hole = 0; /* first position to try */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_map* map;
tommy_node* j;
/* check if the disk is already mapped */
for (j = state->maplist; j != 0; j = j->next) {
map = j->data;
if (strcmp(disk->name, map->name) == 0) {
/* mapping found */
break;
}
}
if (j != 0) {
/* mapping is present, then copy the free blocks into to disk */
disk->total_blocks = map->total_blocks;
disk->free_blocks = map->free_blocks;
continue;
}
/* mapping not found, search for an hole */
while (1) {
for (j = state->maplist; j != 0; j = j->next) {
map = j->data;
if (map->position == hole) {
/* position already used */
break;
}
}
if (j == 0) {
/* hole found */
break;
}
/* try with the next one */
++hole;
}
/* insert the new mapping */
map = map_alloc(disk->name, hole, 0, 0, "");
tommy_list_insert_tail(&state->maplist, &map->node, map);
}
/* without configuration don't check for number of data disks or uuid changes */
if (state->no_conf)
return;
/* counter for the number of UUID mismatches */
uuid_mismatch = 0;
/* check if mapping match the disk uuid */
if (!state->opt.skip_disk_access) {
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
disk = find_disk_by_name(state, map->name);
if (disk == 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: Mapping not found for '%s'\n", map->name);
os_abort();
/* LCOV_EXCL_STOP */
}
if (disk->has_unsupported_uuid) {
/* if uuid is not available, skip this one */
continue;
}
/* if the uuid is changed */
if (strcmp(disk->uuid, map->uuid) != 0) {
/* mark the disk as with an UUID change */
disk->has_different_uuid = 1;
/* if the previous uuid is available */
if (map->uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
log_error(ESOFT, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, disk->uuid);
} else {
/*
* No message here, because having a disk without
* UUID is the normal state of an empty disk
*/
disk->had_empty_uuid = 1;
}
/* update the uuid in the mapping, */
pathcpy(map->uuid, sizeof(map->uuid), disk->uuid);
/* write the new state with the new uuid */
state->need_write = 1;
}
}
}
/* check the parity uuid */
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
for (s = 0; s < state->parity[l].split_mac; ++s) {
char uuid[UUID_MAX];
int ret;
ret = devuuid(state->parity[l].split_map[s].device, state->parity[l].split_map[s].path, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore */
continue;
}
/* if the uuid is changed */
if (strcmp(uuid, state->parity[l].split_map[s].uuid) != 0) {
/* if the previous uuid is available */
if (state->parity[l].split_map[s].uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
log_error(ESOFT, "UUID change for parity '%s/%u' from '%s' to '%s'\n", lev_config_name(l), s, state->parity[l].split_map[s].uuid, uuid);
}
/* update the uuid */
pathcpy(state->parity[l].split_map[s].uuid, sizeof(state->parity[l].split_map[s].uuid), uuid);
/* write the new state with the new uuid */
state->need_write = 1;
}
}
}
}
if (!state->opt.force_uuid && uuid_mismatch > state->level) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Too many disks have changed UUIDs since the last 'sync'.\n");
log_fatal(ESOFT, "If this happened because you actually replaced them,\n");
log_fatal(ESOFT, "you can still '%s', using 'snapraid --force-uuid %s'.\n", state->command, state->command);
log_fatal(ESOFT, "Alternatively, you may have misconfigured the disk mount points,\n");
log_fatal(ESOFT, "and you need to restore the mount points to the state of the latest sync.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* count the number of data disks, including holes left after removing some */
diskcount = 0;
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
if (map->position + 1 > diskcount)
diskcount = map->position + 1;
}
/* ensure to don't go over the limit of the RAID engine */
if (diskcount > RAID_DATA_MAX) {
/* LCOV_EXCL_START */
log_fatal(ESOFT, "Too many data disks. Maximum allowed is %u.\n", RAID_DATA_MAX);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* now count the real number of data disks, excluding holes left after removing some */
diskcount = tommy_list_count(&state->maplist);
/* recommend number of parities */
if (!state->opt.no_warnings) {
/* LCOV_EXCL_START */
if (diskcount >= 36 && state->level < 6) {
log_info(EUSER, "WARNING! For %u disks, it's recommended to use six parity levels.\n", diskcount);
} else if (diskcount >= 29 && state->level < 5) {
log_info(EUSER, "WARNING! For %u disks, it's recommended to use five parity levels.\n", diskcount);
} else if (diskcount >= 22 && state->level < 4) {
log_info(EUSER, "WARNING! For %u disks, it's recommended to use four parity levels.\n", diskcount);
} else if (diskcount >= 15 && state->level < 3) {
log_info(EUSER, "WARNING! For %u disks, it's recommended to use three parity levels.\n", diskcount);
} else if (diskcount >= 5 && state->level < 2) {
log_info(EUSER, "WARNING! For %u disks, it's recommended to use two parity levels.\n", diskcount);
}
/* LCOV_EXCL_STOP */
}
}
void state_refresh(struct snapraid_state* state)
{
tommy_node* i;
unsigned l, s;
uint64_t bs = (uint64_t)state->block_size;
/* for all disks */
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
uint64_t total_space;
uint64_t free_space;
int ret;
disk = find_disk_by_name(state, map->name);
if (disk == 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: Mapping not found for '%s'\n", map->name);
os_abort();
/* LCOV_EXCL_STOP */
}
if (disk->skip_access)
continue;
ret = fsinfo(disk->mount_point, 0, 0, &total_space, &free_space, disk->fstype, sizeof(disk->fstype), disk->fslabel, sizeof(disk->fslabel));
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing disk '%s' to retrieve file-system info. %s.\n", disk->mount_point, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* set the new free blocks */
map->total_blocks = total_space / bs;
map->free_blocks = free_space / bs;
/* also update the disk info */
disk->total_blocks = map->total_blocks;
disk->free_blocks = map->free_blocks;
log_tag("fsinfo_data_split:%s:%" PRIu64 ":%" PRIu64 ":%s:%s\n", esc_tag(disk->name), disk->total_blocks * bs, disk->free_blocks * bs, disk->fstype, disk->fslabel);
log_tag("fsinfo_data:%s:%" PRIu64 ":%" PRIu64 "\n", esc_tag(disk->name), disk->total_blocks * bs, disk->free_blocks * bs);
}
/* for all parities */
for (l = 0; l < state->level; ++l) {
/* set the new free blocks */
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
if (state->parity[l].skip_access)
continue;
for (s = 0; s < state->parity[l].split_mac; ++s) {
struct snapraid_split* split = &state->parity[l].split_map[s];
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(split->path, 0, 0, &total_space, &free_space, split->fstype, sizeof(split->fstype), split->fslabel, sizeof(split->fslabel));
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing file '%s' to retrieve file-system info. %s.\n", split->path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#ifdef WIN32
/*
* Take into account the space we have to leave free
* to avoid the waring about low space
*/
if (free_space >= WINDOWS_SPACEHOLDER_SIZE)
free_space -= WINDOWS_SPACEHOLDER_SIZE;
else
free_space = 0;
#endif
/* add the new free blocks */
uint64_t split_total_blocks = total_space / bs;
uint64_t split_free_blocks = free_space / bs;
state->parity[l].total_blocks += split_total_blocks;
state->parity[l].free_blocks += split_free_blocks;
if (s == 0)
log_tag("fsinfo_parity_split:%s:%" PRIu64 ":%" PRIu64 ":%s:%s\n", lev_config_name(l), split_total_blocks * bs, split_free_blocks * bs, split->fstype, split->fslabel);
else
log_tag("fsinfo_parity_split:%s/%u:%" PRIu64 ":%" PRIu64 ":%s:%s\n", lev_config_name(l), s, split_total_blocks * bs, split_free_blocks * bs, split->fstype, split->fslabel);
}
log_tag("fsinfo_parity:%s:%" PRIu64 ":%" PRIu64 "\n", lev_config_name(l), state->parity[l].total_blocks * bs, state->parity[l].free_blocks * bs);
}
/* for all extra disks */
for (i = state->extralist; i != 0; i = i->next) {
struct snapraid_extra* extra = i->data;
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(extra->dir, 0, 0, &total_space, &free_space, extra->fstype, sizeof(extra->fstype), extra->fslabel, sizeof(extra->fslabel));
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error accessing disk '%s' to retrieve file-system info. %s.\n", extra->dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
log_tag("fsinfo_extra:%s:%" PRIu64 ":%" PRIu64 "\n", esc_tag(extra->name), total_space, free_space);
}
/*
* Note what we don't set need_write = 1, because we don't want
* to update the content file only for the free space info.
*/
}
/**
* Check the content.
*/
static void state_content_check(struct snapraid_state* state, const char* path)
{
tommy_node* i;
/* check that any map has different name and position */
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
tommy_node* j;
for (j = i->next; j != 0; j = j->next) {
struct snapraid_map* other = j->data;
if (strcmp(map->name, other->name) == 0) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "Conflicting 'map' disk specification in '%s'\n", path);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (map->position == other->position) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "Conflicting 'map' index specification in '%s'\n", path);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
}
/**
* Check if the position is REQUIRED, or we can completely clear it from the state.
*
* Note that position with only DELETED blocks are discharged.
*/
static int fs_position_is_required(struct snapraid_state* state, block_off_t pos)
{
tommy_node* i;
/* check for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = fs_par2block_find(disk, pos);
/* if we have at least one file, the position is needed */
if (block_has_file(block))
return 1;
}
return 0;
}
/**
* Check if the info block is REQUIREQ.
*
* This is used to ensure that we keep the last check used for scrubbing.
* and that we add it when importing old context files.
*
* Note that you can have position without info blocks, for example
* if all the blocks are not synced.
*
* Note also that not requiring an info block, doesn't mean that if present it
* can be discarded.
*/
static int fs_info_is_required(struct snapraid_state* state, block_off_t pos)
{
tommy_node* i;
/* check for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = fs_par2block_find(disk, pos);
/* if we have at least one synced file, the info is required */
if (block_state_get(block) == BLOCK_STATE_BLK)
return 1;
}
return 0;
}
static void fs_position_clear_deleted(struct snapraid_state* state, block_off_t pos)
{
tommy_node* i;
/* check for each disk if block is really used */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = fs_par2block_find(disk, pos);
/* if the block is deleted */
if (block_state_get(block) == BLOCK_STATE_DELETED) {
/* set it to empty */
fs_deallocate(disk, pos);
}
}
}
/**
* Check if a block position in a disk is deleted.
*/
static int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t pos)
{
struct snapraid_block* block = fs_par2block_find(disk, pos);
return block_state_get(block) == BLOCK_STATE_DELETED;
}
/**
* Check if a block parity needs sync
*
* This is the same logic used in "status" to detect an incomplete "sync",
* that ignores invalid block, if they are not used by a file in any disk.
*
* This means that DELETED blocks won't necessarily imply an invalid parity.
*/
static int fs_is_block_unsynced(struct snapraid_state* state, block_off_t pos)
{
int one_invalid = 0;
int one_valid = 0;
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
struct snapraid_block* block = fs_par2block_find(disk, pos);
if (block_has_file(block))
one_valid = 1;
if (block_has_invalid_parity(block))
one_invalid = 1;
if (one_invalid && one_valid)
return 1;
}
return 0;
}
/**
* Flush the file checking the final CRC.
* We exploit the fact that the CRC is always stored in the last 4 bytes.
*/
static void decoding_error(const char* path, STREAM* f)
{
unsigned char buf[4];
uint32_t crc_stored;
uint32_t crc_computed;
if (seof(f)) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "Unexpected end of content file '%s' at offset %" PRIi64 "\n", path, stell(f));
log_fatal(ECONTENT, "This content file is truncated. Please use an alternate copy.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
log_fatal(ECONTENT, "Error decoding '%s' at offset %" PRIi64 "\n", path, stell(f));
if (sdeplete(f, buf) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Failed to flush content file '%s' at offset %" PRIi64 "\n", path, stell(f));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* get the stored crc from the last four bytes */
crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
/* get the computed crc */
crc_computed = scrc(f);
/* adjust the stored crc to include itself */
crc_stored = crc32c(crc_stored, buf, 4);
if (crc_computed != crc_stored) {
log_fatal(ECONTENT, "CRC mismatch in '%s'\n", path);
log_fatal(ECONTENT, "This content file is damaged! Please use an alternate copy.\n");
exit(EXIT_FAILURE);
} else {
log_fatal(ECONTENT, "The CRC of the file is correct!\n");
}
}
static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f)
{
block_off_t blockmax;
unsigned count_file;
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
int crc_checked;
char buffer[PATH_MAX];
int ret;
tommy_array disk_mapping;
uint32_t mapping_max;
tommy_hashdyn bucket_hash;
blockmax = 0;
count_file = 0;
count_hardlink = 0;
count_symlink = 0;
count_dir = 0;
count_bad = 0;
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
crc_checked = 0;
mapping_max = 0;
tommy_array_init(&disk_mapping);
tommy_hashdyn_init(&bucket_hash);
/* mark all disks as single threads */
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
disk->single_thread = 1;
}
ret = sread(f, buffer, 12);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid header!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/*
* File format versions:
* - SNAPCNT1/SnapRAID 4.0 First version.
* - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
* The previous 'm' entry is now deprecated, but supported for importing.
* Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'Q' for multi parity file.
* The previous 'P' entry is now deprecated, but supported for importing.
* - SNAPCNT4/SnapRAID 15.0 Adds entry 'd' for dealloc file.
*/
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT4\n\3\0\0", 12) != 0
) {
/* LCOV_EXCL_START */
if (memcmp(buffer, "SNAPCNT", 7) != 0) {
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid header!\n");
os_abort();
} else {
log_fatal(ECONTENT, "The content file '%s' was generated with a newer version of SnapRAID!\n", path);
exit(EXIT_FAILURE);
}
/* LCOV_EXCL_STOP */
}
while (1) {
int c;
/* read the command */
c = sgetc(f);
if (c == EOF) {
break;
}
if (c == 'f') {
/* file */
char sub[PATH_MAX];
uint64_t v_size;
uint64_t v_mtime_sec;
uint32_t v_mtime_nsec;
uint64_t v_inode;
uint32_t v_idx;
struct snapraid_file* file;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: File mapping index out of range\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
ret = sgetb64(f, &v_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (state->block_size == 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Zero blocksize\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check for impossible file size to avoid to crash for a too big allocation */
if (v_size / state->block_size > blockmax) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: File size too big!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_mtime_sec);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_mtime_nsec);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* STAT_NSEC_INVALID is encoded as 0 */
if (v_mtime_nsec == 0)
v_mtime_nsec = STAT_NSEC_INVALID;
else
--v_mtime_nsec;
ret = sgetb64(f, &v_inode);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, sub, sizeof(sub));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*sub) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Null file!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/* allocate the file */
file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
/* insert the file in the file containers */
tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode));
tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub));
tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec));
tommy_list_insert_tail(&disk->filelist, &file->nodelist, file);
/* read all the blocks */
v_idx = 0;
while (v_idx < file->blockmax) {
block_off_t v_pos;
uint32_t v_count;
/* get the "subcommand */
c = sgetc(f);
ret = sgetb32(f, &v_pos);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_count);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (v_count > file->blockmax || v_idx > file->blockmax - v_count) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Block number out of range\n");
os_abort();
/* LCOV_EXCL_STOP */
}
if (v_count > blockmax || v_pos > blockmax - v_count) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Block size %u/%u!\n", blockmax, v_pos + v_count);
os_abort();
/* LCOV_EXCL_STOP */
}
/* fill the blocks in the run */
while (v_count) {
struct snapraid_block* block = fs_file2block_get(file, v_idx);
switch (c) {
case 'b' :
block_state_set(block, BLOCK_STATE_BLK);
break;
case 'n' :
/* deprecated NEW blocks are converted to CHG ones */
block_state_set(block, BLOCK_STATE_CHG);
break;
case 'g' :
block_state_set(block, BLOCK_STATE_CHG);
break;
case 'p' :
block_state_set(block, BLOCK_STATE_REP);
break;
default :
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid block type!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/* read the hash only for 'blk/chg/rep', and not for 'new' */
if (c != 'n') {
ret = sread(f, block->hash, BLOCK_HASH_SIZE);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
} else {
/* set the ZERO hash for deprecated NEW blocks */
hash_zero_set(block->hash);
}
/*
* If we are disabling the copy optimization
* we want also to clear any already previously stored information
* in other sync commands
*/
if (state->opt.force_nocopy && block_state_get(block) == BLOCK_STATE_REP) {
/* set the hash value to INVALID */
hash_invalid_set(block->hash);
/* convert from REP to CHG block */
block_state_set(block, BLOCK_STATE_CHG);
}
/* set the parity association */
fs_allocate(disk, v_pos, file, v_idx);
/* go to the next block */
++v_idx;
++v_pos;
--v_count;
}
}
/* stat */
++count_file;
} else if (c == 'i') {
/* "inf" command */
snapraid_info info;
uint32_t v_pos;
uint64_t v_oldest;
ret = sgetb64(f, &v_oldest);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
v_pos = 0;
while (v_pos < blockmax) {
int bad;
int rehash;
int justsynced;
uint64_t t64;
uint32_t flag;
uint32_t v_count;
ret = sgetb32(f, &v_count);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (v_count > blockmax || v_pos > blockmax - v_count) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Info size %u/%u!\n", blockmax, v_pos + v_count);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &flag);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* if there is an info */
if ((flag & 1) != 0) {
/* read the time */
ret = sgetb64(f, &t64);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* analyze the flags */
bad = (flag & 2) != 0;
rehash = (flag & 4) != 0;
justsynced = (flag & 8) != 0;
if (bad)
count_bad += v_count;
if (rehash)
count_rehash += v_count;
if (justsynced)
count_unscrubbed += v_count;
if (rehash && state->prevhash == HASH_UNDEFINED) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Missing previous checksum!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
info = info_make(t64 + v_oldest, bad, rehash, justsynced);
bucket_insert(&bucket_hash, t64 + v_oldest, v_count, justsynced);
} else {
info = 0;
}
while (v_count) {
/* insert the info in the array */
info_set(&state->infoarr, v_pos, info);
/* ensure that an info is present only for used positions */
if (fs_info_is_required(state, v_pos)) {
if (!info) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Missing info!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
} else {
/*
* Extra info are accepted for backward compatibility
* they are discarded at the first write
*/
}
if (fs_is_block_unsynced(state, v_pos))
++count_unsynced;
/* go to next block */
++v_pos;
--v_count;
}
}
} else if (c == 'h') {
/* hole */
uint32_t v_pos;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Hole mapping index out of range\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
v_pos = 0;
while (v_pos < blockmax) {
uint32_t v_idx;
uint32_t v_count;
struct snapraid_file* deleted;
ret = sgetb32(f, &v_count);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (v_count > blockmax || v_pos > blockmax - v_count) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Hole size %u/%u!\n", blockmax, v_pos + v_count);
os_abort();
/* LCOV_EXCL_STOP */
}
/* get the sub-command */
c = sgetc(f);
switch (c) {
case 'o' :
/* if it's a run of deleted blocks */
/* allocate a fake deleted file */
deleted = file_alloc(state->block_size, "<deleted>", v_count * (data_off_t)state->block_size, 0, 0, 0, 0);
/* mark the file as deleted */
file_flag_set(deleted, FILE_IS_DELETED);
/* insert it in the list of deleted files */
tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted);
/* process all blocks */
v_idx = 0;
while (v_count) {
struct snapraid_block* block = fs_file2block_get(deleted, v_idx);
/* set the block as deleted */
block_state_set(block, BLOCK_STATE_DELETED);
/* read the hash */
ret = sread(f, block->hash, BLOCK_HASH_SIZE);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* insert the block in the block array */
fs_allocate(disk, v_pos, deleted, v_idx);
/* go to next block */
++v_pos;
++v_idx;
--v_count;
}
break;
case 'O' :
/* go to the next run */
v_pos += v_count;
break;
default :
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid hole type!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
}
} else if (c == 'd') {
/* dealloc */
uint32_t v_count;
uint32_t v_pos;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Dealloc mapping index out of range\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
ret = sgetb32(f, &v_count);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
log_tag("content_info:dealloc:%s:%u\n", esc_tag(disk->name), v_count);
for (v_pos = 0; v_pos < v_count; ++v_pos) {
char sub[PATH_MAX];
uint64_t v_size;
uint64_t v_mtime_sec;
uint32_t v_mtime_nsec;
ret = sgetbs(f, sub, sizeof(sub));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*sub) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Null dealloc!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_mtime_sec);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_mtime_nsec);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* STAT_NSEC_INVALID is encoded as 0 */
if (v_mtime_nsec == 0)
v_mtime_nsec = STAT_NSEC_INVALID;
else
--v_mtime_nsec;
/* allocate the file */
struct snapraid_dealloc* dealloc = dealloc_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec);
log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", esc_tag(disk->name), esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec);
/* read all hashes */
for (block_off_t k = 0; k < dealloc->blockmax; ++k) {
unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE;
ret = sread(f, hash, BLOCK_HASH_SIZE);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
}
/* insert the dealloc in the dealloc containers */
tommy_list_insert_tail(&disk->dealloclist, &dealloc->nodelist, dealloc);
}
} else if (c == 's') {
/* symlink */
char sub[PATH_MAX];
char linkto[PATH_MAX];
struct snapraid_link* slink;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Symlink mapping index out of range\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
ret = sgetbs(f, sub, sizeof(sub));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*sub) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Null symlink!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, linkto, sizeof(linkto));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* allocate the link as symbolic link */
slink = link_alloc(sub, linkto, FILE_IS_SYMLINK);
/* insert the link in the link containers */
tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
/* stat */
++count_symlink;
} else if (c == 'a') {
/* hardlink */
char sub[PATH_MAX];
char linkto[PATH_MAX];
struct snapraid_link* slink;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Hardlink mapping index out of range!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
ret = sgetbs(f, sub, sizeof(sub));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*sub) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Null hardlink!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, linkto, sizeof(linkto));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*linkto) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Empty hardlink '%s'!\n", sub);
os_abort();
/* LCOV_EXCL_STOP */
}
/* allocate the link as hard link */
slink = link_alloc(sub, linkto, FILE_IS_HARDLINK);
/* insert the link in the link containers */
tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub));
tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink);
/* stat */
++count_hardlink;
} else if (c == 'r') {
/* dir */
char sub[PATH_MAX];
struct snapraid_dir* dir;
struct snapraid_disk* disk;
uint32_t mapping;
ret = sgetb32(f, &mapping);
if (ret < 0 || mapping >= mapping_max) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Dir mapping index out of range!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
disk = tommy_array_get(&disk_mapping, mapping);
ret = sgetbs(f, sub, sizeof(sub));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (!*sub) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EINTERNAL, "Internal inconsistency: Null dir!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/* allocate the dir */
dir = dir_alloc(sub);
/* insert the dir in the dir containers */
tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub));
tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir);
/* stat */
++count_dir;
} else if (c == 'c') {
/* get the subcommand */
c = sgetc(f);
switch (c) {
case 'u' :
state->hash = HASH_MURMUR3;
break;
case 'k' :
state->hash = HASH_SPOOKY2;
break;
case 'm' :
state->hash = HASH_METRO;
break;
default :
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid checksum!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/* read the seed */
ret = sread(f, state->hashseed, HASH_MAX);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
} else if (c == 'C') {
/* get the sub-command */
c = sgetc(f);
switch (c) {
case 'u' :
state->prevhash = HASH_MURMUR3;
break;
case 'k' :
state->prevhash = HASH_SPOOKY2;
break;
case 'm' :
state->prevhash = HASH_METRO;
break;
default :
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid checksum!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/* read the seed */
ret = sread(f, state->prevhashseed, HASH_MAX);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
} else if (c == 'z') {
uint32_t block_size;
ret = sgetb32(f, &block_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (block_size == 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Zero 'blocksize' specification in the content file!\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* without configuration, auto assign the block size */
if (state->no_conf) {
state->block_size = block_size;
}
if (block_size != state->block_size) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EUSER, "Mismatching 'blocksize' specification in the content file!\n");
log_fatal(EUSER, "Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else if (c == 'y') {
uint32_t hash_size;
ret = sgetb32(f, &hash_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (hash_size < 2 || hash_size > HASH_MAX) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid 'hashsize' specification in the content file!\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* without configuration, auto assign the block size */
if (state->no_conf) {
BLOCK_HASH_SIZE = hash_size;
}
if ((int)hash_size != BLOCK_HASH_SIZE) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EUSER, "Mismatching 'hashsize' specification in the content file!\n");
log_fatal(EUSER, "Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else if (c == 'x') {
ret = sgetb32(f, &blockmax);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
} else if (c == 'm' || c == 'M') {
struct snapraid_map* map;
char uuid[UUID_MAX];
uint32_t v_pos;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
struct snapraid_disk* disk;
ret = sgetbs(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_pos);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* from SnapRAID 7.0 the 'M' command includes the free space */
if (c == 'M') {
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
log_tag("content_data:%s:%" PRIi64 ":%" PRIi64 "\n",
esc_tag(buffer),
v_total_blocks * (uint64_t)state->block_size,
v_free_blocks * (uint64_t)state->block_size);
} else {
v_total_blocks = 0;
v_free_blocks = 0;
}
/* read the uuid */
ret = sgetbs(f, uuid, sizeof(uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
log_tag("content_data_split:%s:%s\n",
esc_tag(buffer),
uuid);
/* find the disk */
disk = find_disk_by_name(state, buffer);
if (!disk) {
/* search by UUID if renamed */
disk = find_disk_by_uuid(state, uuid);
if (disk) {
log_info(EUSER, "WARNING! Renaming disk '%s' to '%s'\n", buffer, disk->name);
/* write the new state with the new name */
state->need_write = 1;
}
}
if (!disk) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EUSER, "Disk '%s' with uuid '%s' not present in the configuration file!\n", buffer, uuid);
log_fatal(EUSER, "If you have removed it from the configuration file, please restore it\n");
/* if it's a command without UUID, it cannot autorename using UUID */
if (state->opt.skip_disk_access)
log_fatal(EUSER, "If you have renamed it, run 'sync' to update the new name\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
map = map_alloc(disk->name, v_pos, v_total_blocks, v_free_blocks, uuid);
tommy_list_insert_tail(&state->maplist, &map->node, map);
/* insert in the mapping vector */
tommy_array_grow(&disk_mapping, mapping_max + 1);
tommy_array_set(&disk_mapping, mapping_max, disk);
++mapping_max;
} else if (c == 'P') {
/*
* From SnapRAID 7.0 the 'P' command includes the free space
* from SnapRAID 11.0 the 'P' command is deprecated by 'Q'
*/
char v_uuid[UUID_MAX];
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
ret = sgetb32(f, &v_level);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, v_uuid, sizeof(v_uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (v_level >= LEV_MAX) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid parity level '%u' in the configuration file!\n", v_level);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
log_tag("content_parity:%s:%" PRIi64 ":%" PRIi64 "\n",
lev_config_name(v_level),
v_total_blocks * (uint64_t)state->block_size,
v_free_blocks * (uint64_t)state->block_size);
/* auto configure if configuration is missing */
if (state->no_conf) {
if (v_level >= state->level)
state->level = v_level + 1;
}
/* if we use this parity entry */
if (v_level < state->level) {
/* if the configuration has more splits, keep them */
if (state->parity[v_level].split_mac < 1)
state->parity[v_level].split_mac = 1;
/* set the parity info */
pathcpy(state->parity[v_level].split_map[0].uuid, sizeof(state->parity[v_level].split_map[0].uuid), v_uuid);
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
log_tag("content_parity_split:%s:%s:%s:%" PRIi64 "\n",
lev_config_name(v_level), /* P command always has a single split */
state->parity[v_level].split_map[0].uuid,
esc_tag(state->parity[v_level].split_map[0].path),
state->parity[v_level].split_map[0].size);
}
} else if (c == 'Q') {
/* from SnapRAID 11.0 the 'Q' command include size info and multi file support */
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
uint32_t v_split_mac;
unsigned s;
ret = sgetb32(f, &v_level);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_split_mac);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (v_split_mac > SPLIT_MAX) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Too many splits '%u' in the content file!\n", v_split_mac);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (v_level >= LEV_MAX) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid parity level '%u' in the configuration file!\n", v_level);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* auto configure if configuration is missing */
if (state->no_conf) {
if (v_level >= state->level)
state->level = v_level + 1;
if (state->parity[v_level].split_mac < v_split_mac)
state->parity[v_level].split_mac = v_split_mac;
}
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
log_tag("content_parity:%s:%" PRIi64 ":%" PRIi64 "\n",
lev_config_name(v_level),
v_total_blocks * (uint64_t)state->block_size,
v_free_blocks * (uint64_t)state->block_size);
for (s = 0; s < v_split_mac; ++s) {
char v_path[PATH_MAX];
char v_uuid[UUID_MAX];
uint64_t v_size;
ret = sgetbs(f, v_path, sizeof(v_path));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, v_uuid, sizeof(v_uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
ret = sgetb64(f, &v_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
/* if we use this parity entry */
if (v_level < state->level) {
/* if this split was removed from the configuration */
if (s >= state->parity[v_level].split_mac) {
/* if the file is used, we really need it */
if (v_size != 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(EUSER, "Parity '%s' misses used file '%u'!\n", lev_config_name(v_level), s);
log_fatal(EUSER, "If you have removed it from the configuration file, please restore it\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* otherwise we can drop it */
log_info(EUSER, "WARNING! Dropping from '%s' unused split '%u'\n", lev_config_name(v_level), s);
} else {
char parity_name[64];
/* we copy the path only if without configuration file */
if (state->no_conf)
pathcpy(state->parity[v_level].split_map[s].path, sizeof(state->parity[v_level].split_map[s].path), v_path);
/* set the split info */
pathcpy(state->parity[v_level].split_map[s].uuid, sizeof(state->parity[v_level].split_map[s].uuid), v_uuid);
state->parity[v_level].split_map[s].size = v_size;
if (s == 0)
pathcpy(parity_name, sizeof(parity_name), lev_config_name(v_level));
else
pathprint(parity_name, sizeof(parity_name), "%s/%u", lev_config_name(v_level), s);
log_tag("content_parity_split:%s:%s:%s:%" PRIi64 "\n",
parity_name,
state->parity[v_level].split_map[s].uuid,
esc_tag(state->parity[v_level].split_map[s].path),
state->parity[v_level].split_map[s].size);
}
}
}
} else if (c == 'N') {
uint32_t crc_stored;
uint32_t crc_computed;
/* get the crc before reading it from the file */
crc_computed = scrc(f);
ret = sgetble32(f, &crc_stored);
if (ret < 0) {
/* LCOV_EXCL_START */
/* here don't call decoding_error() because it's too late to get the crc */
log_fatal(errno, "Error reading the CRC in '%s' at offset %" PRIi64 "\n", path, stell(f));
log_fatal(errno, "This content file is damaged! Use an alternate copy.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (crc_stored != crc_computed) {
/* LCOV_EXCL_START */
/* here don't call decoding_error() because it's too late to get the crc */
log_fatal(ECONTENT, "CRC mismatch in '%s'\n", path);
log_fatal(ECONTENT, "The content file is damaged! Please use an alternate copy.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
crc_checked = 1;
} else {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal(ECONTENT, "Invalid command '%c'!\n", (char)c);
os_abort();
/* LCOV_EXCL_STOP */
}
}
/* mark all disks as multi threads */
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
disk->single_thread = 0;
}
tommy_array_done(&disk_mapping);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error reading the content file '%s' at offset %" PRIi64 "\n", path, stell(f));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!crc_checked) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "Reached the end of '%s' without finding the expected CRC\n", path);
log_fatal(ECONTENT, "This content file is truncated or damaged! Use an alternate copy.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check the file-system on all disks */
state_fscheck(state, "after read");
/* check that the stored parity size matches the loaded state */
if (blockmax != parity_allocated_size(state)) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: Parity size %u/%u in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f));
if (state->opt.skip_content_check) {
log_fatal(ECONTENT, "Overriding.\n");
blockmax = parity_allocated_size(state);
} else {
exit(EXIT_FAILURE);
}
/* LCOV_EXCL_STOP */
}
msg_verbose("%8u files\n", count_file);
msg_verbose("%8u hardlinks\n", count_hardlink);
msg_verbose("%8u symlinks\n", count_symlink);
msg_verbose("%8u empty dirs\n", count_dir);
log_tag("content_info:file:%u\n", count_file);
log_tag("content_info:hardlink:%u\n", count_hardlink);
log_tag("content_info:symlink:%u\n", count_symlink);
log_tag("content_info:dir_empty:%u\n", count_dir);
log_tag("content_info:block:%u\n", blockmax);
log_tag("content_info:block_bad:%u\n", count_bad);
log_tag("content_info:block_rehash:%u\n", count_rehash);
log_tag("content_info:block_unsynced:%u\n", count_unsynced);
log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
/* store the blocks counters */
state->rehash_blocks = count_rehash;
state->bad_blocks = count_bad;
state->unsynced_blocks = count_unsynced;
state->unscrubbed_blocks = count_unscrubbed;
/* store the bucket info list */
bucket_to_list(&bucket_hash, &state->bucketlist, &state->bucketcount);
tommy_hashdyn_done(&bucket_hash);
}
struct state_write_thread_context {
struct snapraid_state* state;
#if HAVE_MT_WRITE
thread_id_t thread;
#endif
/* input */
block_off_t blockmax;
time_t info_oldest;
time_t info_now;
int info_has_rehash;
STREAM* f;
int first;
/* output (required to postpone the output to the terminal after the latest write) */
uint32_t crc;
unsigned count_file;
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
};
static void* state_write_thread(void* arg)
{
struct state_write_thread_context* context = arg;
struct snapraid_state* state = context->state;
block_off_t blockmax = context->blockmax;
time_t info_oldest = context->info_oldest;
time_t info_now = context->info_now;
int info_has_rehash = context->info_has_rehash;
STREAM* f = context->f;
uint32_t crc;
uint64_t t64;
unsigned count_file;
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
tommy_node* i;
block_off_t idx;
block_off_t begin;
unsigned l, s;
tommy_hashdyn bucket_hash;
count_file = 0;
count_hardlink = 0;
count_symlink = 0;
count_dir = 0;
count_bad = 0;
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
tommy_hashdyn_init(&bucket_hash);
/*
* Force at least version 3 a we want to always store the parity size
*
* If there is a dealloc list, force version 4
*/
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (!tommy_list_empty(&disk->dealloclist))
break;
}
/* write header */
if (i == 0)
swrite("SNAPCNT3\n\3\0\0", 12, f);
else
swrite("SNAPCNT4\n\3\0\0", 12, f);
/* write block size and block max */
sputc('z', f);
sputb32(state->block_size, f);
sputc('x', f);
sputb32(blockmax, f);
/* hash size */
sputc('y', f);
sputb32(BLOCK_HASH_SIZE, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
sputc('c', f);
if (state->hash == HASH_MURMUR3) {
sputc('u', f);
} else if (state->hash == HASH_SPOOKY2) {
sputc('k', f);
} else if (state->hash == HASH_METRO) {
sputc('m', f);
} else {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Unexpected hash when writing the content file '%s'.\n", serrorfile(f));
goto bail;
/* LCOV_EXCL_STOP */
}
swrite(state->hashseed, HASH_MAX, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* previous hash only present */
if (state->prevhash != HASH_UNDEFINED) {
/* if at least one rehash tag found, we have to save the previous hash */
if (info_has_rehash) {
sputc('C', f);
if (state->prevhash == HASH_MURMUR3) {
sputc('u', f);
} else if (state->prevhash == HASH_SPOOKY2) {
sputc('k', f);
} else if (state->prevhash == HASH_METRO) {
sputc('m', f);
} else {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
goto bail;
/* LCOV_EXCL_STOP */
}
swrite(state->prevhashseed, HASH_MAX, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
}
/* for each map */
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
/* find the disk for this mapping */
disk = find_disk_by_name(state, map->name);
if (!disk) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: Unmapped disk '%s'\n", map->name);
goto bail;
/* LCOV_EXCL_STOP */
}
/* save the mapping only if disk is mapped */
if (disk->mapping_idx != -1) {
sputc('M', f);
sputbs(map->name, f);
sputb32(map->position, f);
sputb32(map->total_blocks, f);
sputb32(map->free_blocks, f);
sputbs(map->uuid, f);
if (context->first) {
log_tag("content_data:%s:%" PRIi64 ":%" PRIi64 "\n",
esc_tag(map->name),
map->total_blocks * (uint64_t)state->block_size,
map->free_blocks * (uint64_t)state->block_size);
log_tag("content_data_split:%s:%s\n",
esc_tag(map->name),
map->uuid);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
}
/* for each parity */
for (l = 0; l < state->level; ++l) {
sputc('Q', f);
sputb32(l, f);
sputb32(state->parity[l].total_blocks, f);
sputb32(state->parity[l].free_blocks, f);
if (context->first) {
log_tag("content_parity:%s:%" PRIi64 ":%" PRIi64 "\n",
lev_config_name(l),
state->parity[l].total_blocks * (uint64_t)state->block_size,
state->parity[l].free_blocks * (uint64_t)state->block_size);
}
sputb32(state->parity[l].split_mac, f);
for (s = 0; s < state->parity[l].split_mac; ++s) {
char parity_name[64];
sputbs(state->parity[l].split_map[s].path, f);
sputbs(state->parity[l].split_map[s].uuid, f);
sputb64(state->parity[l].split_map[s].size, f);
if (s == 0)
pathcpy(parity_name, sizeof(parity_name), lev_config_name(l));
else
pathprint(parity_name, sizeof(parity_name), "%s/%u", lev_config_name(l), s);
if (context->first) {
log_tag("content_parity_split:%s:%s:%s:%" PRIi64 "\n",
parity_name,
state->parity[l].split_map[s].uuid,
state->parity[l].split_map[s].path,
state->parity[l].split_map[s].size);
}
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
/* if the disk is not mapped, skip it */
if (disk->mapping_idx < 0)
continue;
/* for each file */
for (j = disk->filelist; j != 0; j = j->next) {
struct snapraid_file* file = j->data;
uint64_t size;
uint64_t mtime_sec;
int32_t mtime_nsec;
uint64_t inode;
size = file->size;
mtime_sec = file->mtime_sec;
mtime_nsec = file->mtime_nsec;
inode = file->inode;
sputc('f', f);
sputb32(disk->mapping_idx, f);
sputb64(size, f);
sputb64(mtime_sec, f);
/* encode STAT_NSEC_INVALID as 0 */
if (mtime_nsec == STAT_NSEC_INVALID)
sputb32(0, f);
else
sputb32(mtime_nsec + 1, f);
sputb64(inode, f);
sputbs(file->sub, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* for all the blocks of the file */
begin = 0;
while (begin < file->blockmax) {
unsigned v_state = block_state_get(fs_file2block_get(file, begin));
block_off_t v_pos = fs_file2par_get(disk, file, begin);
uint32_t v_count;
block_off_t end;
/* find the end of run of blocks */
end = begin + 1;
while (end < file->blockmax) {
if (v_state != block_state_get(fs_file2block_get(file, end)))
break;
if (v_pos + (end - begin) != fs_file2par_get(disk, file, end))
break;
++end;
}
switch (v_state) {
case BLOCK_STATE_BLK :
sputc('b', f);
break;
case BLOCK_STATE_CHG :
sputc('g', f);
break;
case BLOCK_STATE_REP :
sputc('p', f);
break;
default :
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: State for block %u state %u\n", v_pos, v_state);
goto bail;
/* LCOV_EXCL_STOP */
}
sputb32(v_pos, f);
v_count = end - begin;
sputb32(v_count, f);
/* write hashes */
for (idx = begin; idx < end; ++idx) {
struct snapraid_block* block = fs_file2block_get(file, idx);
swrite(block->hash, BLOCK_HASH_SIZE, f);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* next begin position */
begin = end;
}
++count_file;
}
/* for each link */
for (j = disk->linklist; j != 0; j = j->next) {
struct snapraid_link* slink = j->data;
switch (link_flag_get(slink, FILE_IS_LINK_MASK)) {
case FILE_IS_HARDLINK :
sputc('a', f);
++count_hardlink;
break;
case FILE_IS_SYMLINK :
sputc('s', f);
++count_symlink;
break;
}
sputb32(disk->mapping_idx, f);
sputbs(slink->sub, f);
sputbs(slink->linkto, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
/* for each dir */
for (j = disk->dirlist; j != 0; j = j->next) {
struct snapraid_dir* dir = j->data;
sputc('r', f);
sputb32(disk->mapping_idx, f);
sputbs(dir->sub, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
++count_dir;
}
/* deleted blocks of the disk */
sputc('h', f);
sputb32(disk->mapping_idx, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
begin = 0;
while (begin < blockmax) {
int is_deleted;
block_off_t end;
is_deleted = fs_is_block_deleted(disk, begin);
/* find the end of run of blocks */
end = begin + 1;
while (end < blockmax
&& is_deleted == fs_is_block_deleted(disk, end)
) {
++end;
}
sputb32(end - begin, f);
if (is_deleted) {
/* write the run of deleted blocks with hash */
sputc('o', f);
/* write all the hash */
while (begin < end) {
struct snapraid_block* block = fs_par2block_get(disk, begin);
swrite(block->hash, BLOCK_HASH_SIZE, f);
++begin;
}
} else {
/*
* Write the run of blocks without hash
* they can be either used or empty blocks
*/
sputc('O', f);
/* next begin position */
begin = end;
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
}
/* deallocated files of the disk */
if (!tommy_list_empty(&disk->dealloclist)) {
sputc('d', f);
sputb32(disk->mapping_idx, f);
uint32_t v_count = tommy_list_count(&disk->dealloclist);
sputb32(v_count, f);
log_tag("content_info:dealloc:%s:%u\n", esc_tag(disk->name), v_count);
/* for each file */
for (j = tommy_list_head(&disk->dealloclist); j != 0; j = j->next) {
struct snapraid_dealloc* dealloc = j->data;
sputbs(dealloc->sub, f);
sputb64(dealloc->size, f);
sputb64(dealloc->mtime_sec, f);
/* encode STAT_NSEC_INVALID as 0 */
if (dealloc->mtime_nsec == STAT_NSEC_INVALID)
sputb32(0, f);
else
sputb32(dealloc->mtime_nsec + 1, f);
log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", esc_tag(disk->name), esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec);
/* write all hashes */
for (block_off_t k = 0; k < dealloc->blockmax; ++k) {
unsigned char* hash = dealloc->blockhash + k * BLOCK_HASH_SIZE;
swrite(hash, BLOCK_HASH_SIZE, f);
}
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
} else {
log_tag("content_info:dealloc:%s:0\n", esc_tag(disk->name));
}
}
/* write the info for each block */
sputc('i', f);
/* ensure to write a 64 bit time */
t64 = info_oldest;
sputb64(t64, f);
begin = 0;
while (begin < blockmax) {
snapraid_info info;
block_off_t end;
block_off_t count;
time_t t;
unsigned flag;
info = info_get(&state->infoarr, begin);
/* avoid this slow operation if not needed */
if (context->first && fs_is_block_unsynced(state, begin))
++count_unsynced;
/* find the end of run of blocks */
end = begin + 1;
while (end < blockmax
&& info == info_get(&state->infoarr, end)
) {
if (context->first && fs_is_block_unsynced(state, end))
++count_unsynced;
++end;
}
count = end - begin;
sputb32(count, f);
/* if there is info */
if (info) {
/* other flags */
flag = 1; /* info is present */
if (info_get_bad(info)) {
flag |= 2;
if (context->first)
count_bad += count;
}
if (info_get_rehash(info)) {
flag |= 4;
if (context->first)
count_rehash += count;
}
if (info_get_justsynced(info)) {
flag |= 8;
if (context->first)
count_unscrubbed += count;
}
sputb32(flag, f);
t = info_get_time(info);
if (context->first)
bucket_insert(&bucket_hash, t, count, flag & 8);
/* truncate any time that is in the future */
if (t > info_now)
t = info_now;
/* the oldest info is computed only on required blocks, so it may not be the absolute oldest */
if (t < info_oldest)
t = 0;
else
t -= info_oldest;
/* ensure to write a 64 bit time */
t64 = t;
sputb64(t64, f);
} else {
/* write a special 0 flag to mark missing info */
sputb32(0, f);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* next begin position */
begin = end;
}
sputc('N', f);
/* flush data written to the disk */
if (sflush(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s' (in flush before crc). %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* get the file crc */
crc = scrc(f);
/*
* Compare the crc of the data written to file
* with the one of the data written to the stream
*/
if (crc != scrc_stream(f)) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "CRC mismatch while writing the content stream.\n");
log_fatal(ECONTENT, "DANGER! Your RAM memory is faulty! DO NOT PROCEED UNTIL FIXED!\n");
log_fatal(ECONTENT, "Try running a memory test like http://www.memtest86.com/\n");
goto bail;
/* LCOV_EXCL_STOP */
}
sputble32(crc, f);
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
goto bail;
/* LCOV_EXCL_STOP */
}
/* set output variables */
context->crc = crc;
context->count_file = count_file;
context->count_hardlink = count_hardlink;
context->count_symlink = count_symlink;
context->count_dir = count_dir;
context->count_bad = count_bad;
context->count_rehash = count_rehash;
context->count_unsynced = count_unsynced;
context->count_unscrubbed = count_unscrubbed;
/* store the bucket info list */
if (context->first)
bucket_to_list(&bucket_hash, &state->bucketlist, &state->bucketcount);
tommy_hashdyn_done(&bucket_hash);
return 0;
bail:
tommy_hashdyn_done(&bucket_hash);
return context;
}
static void state_write_content(struct snapraid_state* state, uint32_t* out_crc)
{
#if HAVE_MT_WRITE
int fail;
int first;
#else
STREAM* f;
unsigned count_content;
unsigned k;
struct state_write_thread_context* context;
void* retval;
#endif
tommy_node* i;
block_off_t blockmax;
time_t info_oldest;
time_t info_now;
int info_has_rehash;
int mapping_idx;
block_off_t idx;
uint32_t crc;
unsigned count_file;
unsigned count_hardlink;
unsigned count_symlink;
unsigned count_dir;
unsigned count_bad;
unsigned count_rehash;
unsigned count_unsynced;
unsigned count_unscrubbed;
/* blocks of all array */
blockmax = parity_allocated_size(state);
/* check the file-system on all disks */
state_fscheck(state, "before write");
/*
* Clear the info for unused blocks
* and get some other info
*/
info_oldest = 0; /* oldest time in info */
info_now = time(0); /* get the present time */
info_has_rehash = 0; /* if there is a rehash info */
for (idx = 0; idx < blockmax; ++idx) {
/* if the position is used */
if (fs_position_is_required(state, idx)) {
snapraid_info info = info_get(&state->infoarr, idx);
/* only if there is some info to store */
if (info) {
time_t info_time = info_get_time(info);
if (!info_oldest || info_time < info_oldest)
info_oldest = info_time;
if (info_get_rehash(info))
info_has_rehash = 1;
}
} else {
/* clear any previous info */
info_set(&state->infoarr, idx, 0);
/* and clear any deleted blocks */
fs_position_clear_deleted(state, idx);
}
}
if (info_has_rehash && state->prevhash == HASH_UNDEFINED) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: Rehash blocks found but previous checksum is missing!\n");
os_abort();
/* LCOV_EXCL_STOP */
}
/* map disks */
mapping_idx = 0;
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
/* find the disk for this mapping */
disk = find_disk_by_name(state, map->name);
if (!disk) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: Unmapped disk '%s'\n", map->name);
os_abort();
/* LCOV_EXCL_STOP */
}
/* save the mapping only for not empty disks */
if (!fs_is_empty(disk, blockmax)) {
/* assign the mapping index used to identify disks */
disk->mapping_idx = mapping_idx;
++mapping_idx;
} else {
/* mark the disk as without mapping */
disk->mapping_idx = -1;
}
}
#if HAVE_MT_WRITE
/* start all writing threads */
first = 1;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
struct state_write_thread_context* context;
char tmp[PATH_MAX];
STREAM* f;
msg_progress("Saving state to %s...\n", content->content);
log_tag("content_write:%s\n", esc_tag(content->content));
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
/* ensure to delete a previous stale file */
if (remove(tmp) != 0) {
if (errno != ENOENT) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
f = sopen_write(tmp);
if (f == 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* allocate the thread context */
context = malloc_nofail(sizeof(struct state_write_thread_context));
content->context = context;
/* initialize */
context->state = state;
context->blockmax = blockmax;
context->info_oldest = info_oldest;
context->info_now = info_now;
context->info_has_rehash = info_has_rehash;
context->f = f;
context->first = first;
first = 0;
thread_create(&context->thread, state_write_thread, context);
i = i->next;
}
/* join all thread */
fail = 0;
first = 1;
crc = 0;
count_file = 0;
count_hardlink = 0;
count_symlink = 0;
count_dir = 0;
count_bad = 0;
count_rehash = 0;
count_unsynced = 0;
count_unscrubbed = 0;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
struct state_write_thread_context* context = content->context;
void* retval;
thread_join(context->thread, &retval);
if (retval) {
/* LCOV_EXCL_START */
fail = 1;
/* LCOV_EXCL_STOP */
} else {
STREAM* f = context->f;
/*
* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure
* than even in a system crash event we have one valid copy of the file.
*/
if (sflush(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#if HAVE_FSYNC
if (ssync(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
if (sclose(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error closing the content file. %s.\n", strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (first) {
first = 0;
crc = context->crc;
count_file = context->count_file;
count_hardlink = context->count_hardlink;
count_symlink = context->count_symlink;
count_dir = context->count_dir;
cound_bad = context->count_bad;
count_rehash = context->count_rehash;
count_unsynced = context->count_unsynced;
count_unscrubbed = context->count_unscrubbed;
} else {
if (crc != context->crc) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "Different CRCs writing content streams.\n");
log_fatal(ECONTENT, "DANGER! Your RAM memory is broken! DO NOT PROCEED UNTIL FIXED!\n");
log_fatal(ECONTENT, "Try running a memory test like http://www.memtest86.com/\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
free(context);
i = i->next;
}
/* abort on failure */
if (fail) {
/* LCOV_EXCL_START */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#else
/* count the content files */
count_content = 0;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
msg_progress("Saving state to %s...\n", content->content);
++count_content;
i = i->next;
}
/* open all the content files */
f = sopen_multi_write(count_content, STREAM_FLAGS_SEQUENTIAL | STREAM_FLAGS_CRC);
if (!f) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error opening the content files.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
k = 0;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
char tmp[PATH_MAX];
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
/* ensure to delete a previous stale file */
if (remove(tmp) != 0) {
if (errno != ENOENT) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error removing the stale content file '%s'. %s.\n", tmp, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
if (sopen_multi_file(f, k, tmp) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error opening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
++k;
i = i->next;
}
/* allocate the thread context */
context = malloc_nofail(sizeof(struct state_write_thread_context));
/* initialize */
context->state = state;
context->blockmax = blockmax;
context->info_oldest = info_oldest;
context->info_now = info_now;
context->info_has_rehash = info_has_rehash;
context->f = f;
context->first = 1;
retval = state_write_thread(context);
/* abort on failure */
if (retval) {
/* LCOV_EXCL_START */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/*
* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure
* than even in a system crash event we have one valid copy of the file.
*/
if (sflush(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s', in flush(). %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#if HAVE_FSYNC
if (ssync(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error writing the content file '%s' in sync(). %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
if (sclose(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error closing the content file. %s.\n", strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
crc = context->crc;
count_file = context->count_file;
count_hardlink = context->count_hardlink;
count_symlink = context->count_symlink;
count_dir = context->count_dir;
count_bad = context->count_bad;
count_rehash = context->count_rehash;
count_unsynced = context->count_unsynced;
count_unscrubbed = context->count_unscrubbed;
free(context);
#endif
msg_verbose("%8u files\n", count_file);
msg_verbose("%8u hardlinks\n", count_hardlink);
msg_verbose("%8u symlinks\n", count_symlink);
msg_verbose("%8u empty dirs\n", count_dir);
log_tag("content_info:file:%u\n", count_file);
log_tag("content_info:hardlink:%u\n", count_hardlink);
log_tag("content_info:symlink:%u\n", count_symlink);
log_tag("content_info:dir_empty:%u\n", count_dir);
log_tag("content_info:block:%u\n", blockmax);
log_tag("content_info:block_bad:%u\n", count_bad);
log_tag("content_info:block_rehash:%u\n", count_rehash);
log_tag("content_info:block_unsynced:%u\n", count_unsynced);
log_tag("content_info:block_unscrubbed:%u\n", count_unscrubbed);
/* store the blocks counters */
state->rehash_blocks = count_rehash;
state->bad_blocks = count_bad;
state->unsynced_blocks = count_unsynced;
state->unscrubbed_blocks = count_unscrubbed;
*out_crc = crc;
}
void state_probe(struct snapraid_state* state)
{
char path[PATH_MAX];
struct stat st;
tommy_node* node;
/* iterate over all the available content files and check the first one present */
node = tommy_list_head(&state->contentlist);
while (node) {
struct snapraid_content* content = node->data;
pathcpy(path, sizeof(path), content->content);
if (stat(path, &st) == 0) {
/* if found stop the search */
break;
}
/* next content file */
node = node->next;
}
/* if not found, assume empty */
if (!node)
return;
if (!state->no_conf) {
/* intentionally don't output the content path because it's not really read */
log_tag("content_info:probe_unixtime:%" PRId64 "\n", (int64_t)st.st_mtime);
log_flush();
}
}
void state_read(struct snapraid_state* state)
{
STREAM* f;
char path[PATH_MAX];
struct stat st;
tommy_node* node;
int ret;
int c;
/* iterate over all the available content files and load the first one present */
f = 0;
node = tommy_list_head(&state->contentlist);
while (node) {
struct snapraid_content* content = node->data;
pathcpy(path, sizeof(path), content->content);
msg_progress("Loading state from %s...\n", path);
f = sopen_read(path, STREAM_FLAGS_SEQUENTIAL | STREAM_FLAGS_CRC);
if (f != 0) {
/* if found stop the search */
break;
} else {
/* if it's real error of an existing file, abort */
if (errno != ENOENT) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error opening the content file '%s'. %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* otherwise continue */
if (node->next) {
log_error(errno, "WARNING! Content file '%s' not found, attempting with another copy...\n", path);
/* ensure to rewrite all the content files */
state->need_write = 1;
}
}
/* next content file */
node = node->next;
}
/* if not found, assume empty */
if (!f) {
log_info(EUSER, "No content file found. Assuming empty.\n");
/* create the initial mapping */
state_map(state);
return;
}
/* get the stat of the content file */
ret = fstat(shandle(f), &st);
if (ret != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error stating the content file '%s'. %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (!state->no_conf) {
log_tag("content:%s\n", esc_tag(path));
log_tag("content_info:read_unixtime:%" PRId64 "\n", (int64_t)st.st_mtime);
log_flush();
}
/* go further to check other content files */
while (node) {
char other_path[PATH_MAX];
struct stat other_st;
struct snapraid_content* content = node->data;
pathcpy(other_path, sizeof(other_path), content->content);
ret = stat(other_path, &other_st);
if (ret != 0) {
/* allow missing content files, but not any other kind of error */
if (errno != ENOENT) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error stating the content file '%s'. %s.\n", other_path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* ensure to rewrite all the content files */
state->need_write = 1;
} else {
/* if the size is different */
if (other_st.st_size != st.st_size) {
log_error(ECONTENT, "WARNING! Content files '%s' and '%s' have a different size!\n", path, other_path);
log_error(ECONTENT, "Likely one of the two is broken!\n");
/* ensure to rewrite all the content files */
state->need_write = 1;
}
}
/* next content file */
node = node->next;
}
/*
* Start with a undefined default.
* it's for compatibility with version 1.0 where MD5 was implicit.
*/
state->hash = HASH_UNDEFINED;
/* start with a zero seed, it was the default in old versions */
memset(state->hashseed, 0, HASH_MAX);
/* previous hash, start with an undefined value */
state->prevhash = HASH_UNDEFINED;
/* intentionally not set the prevhashseed, if used valgrind will warn about it */
/* get the first char to detect the file type */
c = sgetc(f);
sungetc(c, f);
/* guess the file type from the first char */
if (c == 'S') {
state_read_content(state, path, f);
} else {
/* LCOV_EXCL_START */
log_fatal(EUSER, "From SnapRAID v9.0 the text content file is not supported anymore.\n");
log_fatal(EUSER, "You have first to upgrade to SnapRAID v8.1 to convert it to binary format.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
sclose(f);
if (state->hash == HASH_UNDEFINED) {
/* LCOV_EXCL_START */
log_fatal(EUSER, "The checksum to use is not specified.\n");
log_fatal(EUSER, "This happens because you are likely upgrading from SnapRAID 1.0.\n");
log_fatal(EUSER, "To use a new SnapRAID you must restart from scratch,\n");
log_fatal(EUSER, "deleting all the content and parity files.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (state->unsynced_blocks)
msg_progress("WARNING! The latest sync was interrupted!\n");
/* update the mapping */
state_map(state);
state_content_check(state, path);
/* mark that we read the content file, and it passed all the checks */
state->checked_read = 1;
}
struct state_verify_thread_context {
struct snapraid_state* state;
struct snapraid_content* content;
#if HAVE_MT_VERIFY
thread_id_t thread;
#else
void* retval;
#endif
/* input */
uint32_t crc;
STREAM* f;
};
static void* state_verify_thread(void* arg)
{
struct state_verify_thread_context* context = arg;
struct snapraid_content* content = context->content;
STREAM* f = context->f;
unsigned char buf[4];
uint32_t crc_stored;
uint32_t crc_computed;
uint64_t start;
start = os_tick_ms();
if (sdeplete(f, buf) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Failed to flush content file '%s'. %s.\n", serrorfile(f), strerror(errno));
return context;
/* LCOV_EXCL_STOP */
}
/* get the stored crc from the last four bytes */
crc_stored = buf[0] | (uint32_t)buf[1] << 8 | (uint32_t)buf[2] << 16 | (uint32_t)buf[3] << 24;
if (crc_stored != context->crc) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "DANGER! Wrong stored CRC in '%s'\n", serrorfile(f));
return context;
/* LCOV_EXCL_STOP */
}
/* get the computed crc */
crc_computed = scrc(f);
/* adjust the stored crc to include itself */
crc_stored = crc32c(crc_stored, buf, 4);
if (crc_computed != crc_stored) {
/* LCOV_EXCL_START */
log_fatal(ECONTENT, "DANGER! Wrong file CRC in '%s'\n", serrorfile(f));
return context;
/* LCOV_EXCL_STOP */
}
msg_progress("Verified %s in %" PRIu64 " seconds\n", content->content, (os_tick_ms() - start) / 1000);
return 0;
}
static void state_verify_content(struct snapraid_state* state, uint32_t crc)
{
tommy_node* i;
int fail;
msg_progress("Verifying...\n");
/* start all reading threads */
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
struct state_verify_thread_context* context;
char tmp[PATH_MAX];
STREAM* f;
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
f = sopen_read(tmp, STREAM_FLAGS_SEQUENTIAL | STREAM_FLAGS_CRC);
if (f == 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error reopening the temporary content file '%s'. %s.\n", tmp, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* allocate the thread context */
context = malloc_nofail(sizeof(struct state_verify_thread_context));
content->context = context;
/* initialize */
context->state = state;
context->content = content;
context->crc = crc;
context->f = f;
#if HAVE_MT_VERIFY
thread_create(&context->thread, state_verify_thread, context);
#else
context->retval = state_verify_thread(context);
#endif
i = i->next;
}
/* join all thread */
fail = 0;
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
struct state_verify_thread_context* context = content->context;
void* retval;
#if HAVE_MT_VERIFY
thread_join(context->thread, &retval);
#else
retval = context->retval;
#endif
if (retval) {
/* LCOV_EXCL_START */
fail = 1;
/* LCOV_EXCL_STOP */
} else {
STREAM* f = context->f;
if (sclose(f) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error closing the content file. %s.\n", strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
free(context);
i = i->next;
}
/* abort on failure */
if (fail) {
/* LCOV_EXCL_START */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
static void state_rename_content(struct snapraid_state* state)
{
tommy_node* i;
#if defined(__linux__) /* this sequence is linux specific */
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
char tmp[PATH_MAX];
char dir[PATH_MAX];
char* slash;
int handle;
pathcpy(dir, sizeof(dir), content->content);
slash = strrchr(dir, '/');
if (slash)
slash[1] = 0;
else
pathcpy(dir, sizeof(dir), ".");
/* open the directory to get the handle */
handle = open(dir, O_RDONLY | O_DIRECTORY);
if (handle < 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error opening the directory '%s'. %s.\n", dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* now rename the just written copy with the correct name */
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
if (rename(tmp, content->content) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error renaming the content file '%s' to '%s'. %s.\n", tmp, content->content, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* sync the directory */
if (fsync(handle) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error syncing the directory '%s'. %s.\n", dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (close(handle) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error closing the directory '%s'. %s.\n", dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
i = i->next;
}
#else
i = tommy_list_head(&state->contentlist);
while (i) {
struct snapraid_content* content = i->data;
char tmp[PATH_MAX];
/* now renames the just written copy with the correct name */
pathprint(tmp, sizeof(tmp), "%s.tmp", content->content);
if (rename(tmp, content->content) != 0) {
/* LCOV_EXCL_START */
log_fatal(errno, "Error renaming the content file '%s' to '%s' in rename(). %s.\n", tmp, content->content, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
i = i->next;
}
#endif
}
void state_write(struct snapraid_state* state)
{
uint32_t crc;
time_t now;
/* write all the content files */
state_write_content(state, &crc);
/* verify the just written files */
state_verify_content(state, crc);
/* rename the new files, over the old ones */
state_rename_content(state);
/* log the write time of the content file */
now = time(0);
log_tag("content_info:write_unixtime:%" PRId64 "\n", (int64_t)now);
log_flush();
state->need_write = 0; /* no write needed anymore */
state->checked_read = 0; /* what we wrote is not checked in read */
state->written = 1;
}
void state_commit(struct snapraid_state* state)
{
tommy_node* i;
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
/* clear the dealloc list */
tommy_list_foreach(&disk->dealloclist, (tommy_foreach_func*)dealloc_free);
tommy_list_init(&disk->dealloclist);
}
}
void state_skip(struct snapraid_state* state)
{
tommy_node* i;
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
if (!disk->skip_access)
continue;
/* for each file */
for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
struct snapraid_file* file = j->data;
file_flag_set(file, FILE_IS_EXCLUDED);
}
/* for each link */
for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
struct snapraid_link* slink = j->data;
link_flag_set(slink, FILE_IS_EXCLUDED);
}
/* for each dir */
for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
struct snapraid_dir* dir = j->data;
dir_flag_set(dir, FILE_IS_EXCLUDED);
}
}
}
void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing, int filter_error)
{
tommy_node* i;
unsigned l;
/* if no filter, include all */
if (!filter_missing && !filter_error && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk))
return;
msg_progress("Selecting...\n");
for (i = tommy_list_head(filterlist_disk); i != 0; i = i->next) {
struct snapraid_filter* filter = i->data;
msg_verbose("\t%s%s\n", filter->pattern, filter->is_disk ? "//" : "");
}
for (i = tommy_list_head(filterlist_file); i != 0; i = i->next) {
struct snapraid_filter* filter = i->data;
msg_verbose("\t%s%s\n", filter->pattern, filter->is_dir ? "/" : "");
}
if (filter_missing)
msg_verbose("\t<missing>\n");
if (filter_error)
msg_verbose("\t<error>\n");
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
/* if we filter for presence, we have to access the disk, so better to print something */
if (filter_missing)
msg_progress("Scanning disk %s...\n", disk->name);
/* for each file */
for (j = tommy_list_head(&disk->filelist); j != 0; j = j->next) {
struct snapraid_file* file = j->data;
if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0
|| filter_path(filterlist_file, 0, disk->name, file->sub) != 0
|| filter_existence(filter_missing, disk->dir, file->sub) != 0
|| filter_correctness(filter_error, &state->infoarr, disk, file) != 0
) {
file_flag_set(file, FILE_IS_EXCLUDED);
}
}
/* for each link */
for (j = tommy_list_head(&disk->linklist); j != 0; j = j->next) {
struct snapraid_link* slink = j->data;
if (filter_path(filterlist_disk, 0, disk->name, slink->sub) != 0
|| filter_path(filterlist_file, 0, disk->name, slink->sub) != 0
|| filter_existence(filter_missing, disk->dir, slink->sub) != 0
) {
link_flag_set(slink, FILE_IS_EXCLUDED);
}
}
/* for each empty dir */
for (j = tommy_list_head(&disk->dirlist); j != 0; j = j->next) {
struct snapraid_dir* dir = j->data;
if (filter_emptydir(filterlist_disk, 0, disk->name, dir->sub) != 0
|| filter_emptydir(filterlist_file, 0, disk->name, dir->sub) != 0
|| filter_existence(filter_missing, disk->dir, dir->sub) != 0
) {
dir_flag_set(dir, FILE_IS_EXCLUDED);
}
}
}
/* if we are filtering by disk, exclude any parity not explicitly included */
if (!tommy_list_empty(filterlist_disk)) {
/* for each parity disk */
for (l = 0; l < state->level; ++l) {
/* check if the parity is excluded by name */
if (filter_path(filterlist_disk, 0, lev_config_name(l), 0) != 0) {
/* excluded the parity from further operation */
state->parity[l].is_excluded_by_filter = 1;
}
}
} else {
/* if we are filtering by file, exclude all parity */
if (filter_missing || !tommy_list_empty(filterlist_file)) {
/* for each parity disk */
for (l = 0; l < state->level; ++l) {
state->parity[l].is_excluded_by_filter = 1;
}
}
}
}
int state_progress_begin(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, block_off_t countmax)
{
time_t now;
if (state->opt.gui) {
log_tag("run:begin:%u:%u:%u\n", blockstart, blockmax, countmax);
log_flush();
}
now = time(0);
state->progress_whole_start = now;
state->progress_tick = 0;
state->progress_ptr = 0;
state->progress_wasted = 0;
/* start of thermal control */
if (!state_thermal_begin(state, now))
return -1;
/* stop if requested */
if (global_interrupt) {
/* LCOV_EXCL_START */
if (!state->opt.gui) {
msg_status("Not starting due to interruption\n");
}
log_tag("sigint:0: SIGINT received\n");
log_flush();
return 1;
/* LCOV_EXCL_STOP */
}
return 0;
}
void state_progress_end(struct snapraid_state* state, block_off_t countpos, block_off_t countmax, data_off_t countsize, const char* msg)
{
if (state->opt.gui) {
log_tag("run:end\n");
log_flush();
} else if (countmax == 0) {
if (state->need_write || state->written) {
msg_status("100%% completed\n");
} else {
msg_status("%s", msg);
}
} else {
time_t now;
time_t elapsed;
unsigned countsize_MB = (countsize + MEGA - 1) / MEGA;
now = time(0);
elapsed = now - state->progress_whole_start - state->progress_wasted;
msg_bar("%u%% completed, %u MB accessed", muldiv(countpos, 100, countmax), countsize_MB);
msg_bar(" in %u:%02u", (unsigned)(elapsed / 3600), (unsigned)((elapsed % 3600) / 60));
msg_bar("\n");
msg_flush();
}
}
void state_progress_stop(struct snapraid_state* state)
{
time_t now;
now = time(0);
state->progress_interruption = now;
}
void state_progress_restart(struct snapraid_state* state)
{
time_t now;
now = time(0);
/* reset the progress counter */
state->progress_tick = 0;
state->progress_ptr = 0;
if (now >= state->progress_interruption) /* avoid degenerated cases when the clock is manually adjusted */
state->progress_wasted += now - state->progress_interruption;
}
/**
* Set the latest os_tick data in the progress vector.
*/
static void state_progress_latest(struct snapraid_state* state)
{
tommy_node* i;
unsigned l;
state->progress_tick_misc[state->progress_ptr] = state->tick_misc;
state->progress_tick_sched[state->progress_ptr] = state->tick_sched;
state->progress_tick_raid[state->progress_ptr] = state->tick_raid;
state->progress_tick_hash[state->progress_ptr] = state->tick_hash;
state->progress_tick_io[state->progress_ptr] = state->tick_io;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
disk->progress_tick[state->progress_ptr] = disk->tick;
}
for (l = 0; l < state->level; ++l)
state->parity[l].progress_tick[state->progress_ptr] = state->parity[l].tick;
}
/**
* Number of columns
*/
#define GRAPH_COLUMN 78
/**
* Get the reference value, 0 if index is PROGRESS_MAX
*/
#define ref(map, index) (index < PROGRESS_MAX ? map[index] : 0)
static struct snapraid_thermal* state_thermal_find(struct snapraid_state* state, const char* name)
{
tommy_node* i;
struct snapraid_thermal* found = 0;
for (i = tommy_list_head(&state->thermallist); i != 0; i = i->next) {
struct snapraid_thermal* thermal = i->data;
if (strcmp(thermal->name, name) == 0) {
if (found == 0
/* if multiple matches, return the one with highest temperature */
|| found->latest_temperature < thermal->latest_temperature
/* or, if the temperature is equal, the one with the higher r_squared */
|| (found->latest_temperature == thermal->latest_temperature && found->params.r_squared < thermal->params.r_squared)
) {
found = thermal;
}
}
}
return found;
}
#define THERMAL_PAD " "
static void state_progress_graph(struct snapraid_state* state, struct snapraid_io* io, unsigned current, unsigned oldest)
{
uint64_t v;
uint64_t tick_total;
unsigned bar;
tommy_node* i;
unsigned l;
size_t pad;
size_t pre;
tick_total = 0;
tick_total += state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
tick_total += state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
tick_total += state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
tick_total += state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
tick_total += state->progress_tick_io[current] - ref(state->progress_tick_io, oldest);
if (!tick_total)
return;
pad = 4;
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
size_t len = strlen(disk->name);
if (pad < len)
pad = len;
}
for (l = 0; l < state->level; ++l) {
size_t len = strlen(lev_config_name(l));
if (pad < len)
pad = len;
}
/* extra space */
pad += 1;
pre = 4;
if (pad + pre + 30 < GRAPH_COLUMN)
bar = GRAPH_COLUMN - pad - pre;
else
bar = 30; /* at least a bar of 30 */
if (io) {
const char* legend = "cached blocks (instant, more is better)";
/* refresh the cached blocks info */
io_refresh(io);
printf("\n");
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
v = disk->cached_blocks;
printr(disk->name, pad);
printf("%4" PRIu64 " | ", v);
if (disk->progress_file && disk->progress_file->sub)
printf("%s", disk->progress_file->sub);
else
printf("-");
printf("\n");
}
for (l = 0; l < state->level; ++l) {
v = state->parity[l].cached_blocks;
printr(lev_config_name(l), pad);
printf("%4" PRIu64 " | ", v);
printc('o', v * bar / io->io_max);
printf("\n");
}
printc(' ', pad);
printf(" |_");
printc('_', bar);
printf("\n");
printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
printf("%s", legend);
printf("\n");
}
printf("\n");
pre = 4;
if (state->thermal_temperature_limit != 0)
pre += strlen(THERMAL_PAD);
if (pad + pre + 30 < GRAPH_COLUMN)
bar = GRAPH_COLUMN - pad - pre;
else
bar = 30; /* at least a bar of 30 */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
v = disk->progress_tick[current] - ref(disk->progress_tick, oldest);
printr(disk->name, pad);
if (state->thermal_temperature_limit != 0) {
struct snapraid_thermal* thermal = state_thermal_find(state, disk->name);
if (thermal) {
if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
printf(" %2u (%2u)", thermal->latest_temperature, (int)thermal->params.t_steady);
else
printf(" %2u ", thermal->latest_temperature);
} else {
printf(THERMAL_PAD);
}
}
printf("%3u%% | ", muldiv(v, 100, tick_total));
printc('*', v * bar / tick_total);
printf("\n");
/* clear the file in progress */
disk->progress_file = 0;
}
for (l = 0; l < state->level; ++l) {
v = state->parity[l].progress_tick[current] - ref(state->parity[l].progress_tick, oldest);
printr(lev_config_name(l), pad);
if (state->thermal_temperature_limit != 0) {
struct snapraid_thermal* thermal = state_thermal_find(state, lev_config_name(l));
if (thermal) {
if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
printf(" %2u (%2u)", thermal->latest_temperature, (int)thermal->params.t_steady);
else
printf(" %2u ", thermal->latest_temperature);
} else {
printf(THERMAL_PAD);
}
}
printf("%3u%% | ", muldiv(v, 100, tick_total));
printc('*', v * bar / tick_total);
printf("\n");
}
v = state->progress_tick_raid[current] - ref(state->progress_tick_raid, oldest);
printr("raid", pad);
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printf("%3u%% | ", muldiv(v, 100, tick_total));
printc('*', v * bar / tick_total);
printf("\n");
v = state->progress_tick_hash[current] - ref(state->progress_tick_hash, oldest);
printr("hash", pad);
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printf("%3u%% | ", muldiv(v, 100, tick_total));
printc('*', v * bar / tick_total);
printf("\n");
v = state->progress_tick_sched[current] - ref(state->progress_tick_sched, oldest);
printr("sched", pad);
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printf("%3u%% | ", muldiv(v, 100, tick_total));
printc('*', v * bar / tick_total);
printf("\n");
v = state->progress_tick_misc[current] - ref(state->progress_tick_misc, oldest);
printr("misc", pad);
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printf("%3u%% | ", muldiv(v, 100, tick_total));
printc('*', v * bar / tick_total);
printf("\n");
printc(' ', pad);
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printf(" |_");
printc('_', bar);
printf("\n");
if (oldest == PROGRESS_MAX) {
const char* legend = "wait time (total, less is better)";
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend) / 2);
printf("%s", legend);
printf("\n");
} else {
const char* legend_d = "wait time (last %d secs, less is better)";
if (state->thermal_temperature_limit != 0)
printf(THERMAL_PAD);
printc(' ', 5 + pad + 1 + bar / 2 - strlen(legend_d) / 2);
printf(legend_d, PROGRESS_MAX);
printf("\n");
}
printf("\n");
}
int state_progress(struct snapraid_state* state, struct snapraid_io* io, block_off_t blockpos, block_off_t countpos, block_off_t countmax, data_off_t countsize)
{
time_t now;
int pred;
now = time(0);
/* thermal measure */
if (now > state->thermal_latest + THERMAL_PERIOD_SECONDS || state->opt.fake_device) {
state->thermal_latest = now;
state_thermal(state, now);
}
/* previous position */
pred = state->progress_ptr + PROGRESS_MAX - 1;
if (pred >= PROGRESS_MAX)
pred -= PROGRESS_MAX;
/* if the previous measure is different */
if (state->progress_tick == 0
|| state->progress_time[pred] != now
) {
time_t elapsed;
unsigned out_perc = 0;
unsigned out_size_speed = 0;
unsigned out_block_speed = 0;
unsigned out_cpu = 0;
unsigned out_eta = 0;
int out_temperature = 0;
int out_steady = 0;
int out_computed = 0;
/* store the new measure */
state->progress_time[state->progress_ptr] = now;
state->progress_pos[state->progress_ptr] = countpos;
state->progress_size[state->progress_ptr] = countsize;
state_progress_latest(state);
elapsed = now - state->progress_whole_start - state->progress_wasted;
/* completion percentage */
if (countmax)
out_perc = muldiv(countpos, 100, countmax);
/* if we have at least 5 measures */
if (state->progress_tick >= 5
/* or if we are running in test mode, with at least one measure */
|| (state->opt.force_progress && state->progress_tick >= 1)
) {
int oldest;
int past;
time_t delta_time;
block_off_t delta_pos;
data_off_t delta_size;
uint64_t tick_cpu;
uint64_t tick_total;
uint64_t delta_tick_cpu;
uint64_t delta_tick_total;
uint64_t oldest_tick_cpu;
uint64_t oldest_tick_total;
tommy_node* i;
/* number of past measures */
past = state->progress_tick;
/*
* Drop the oldest ones, to promptly
* skip the startup phase
*/
past -= past / 5;
/* check how much we can go in the past */
if (past >= PROGRESS_MAX - 1) {
/*
* The vector is filled, so we are already in position
* to get the possible oldest one
*/
oldest = state->progress_ptr + 1;
} else {
/* go backward the number of positions selected */
oldest = state->progress_ptr + PROGRESS_MAX - past;
}
if (oldest >= PROGRESS_MAX)
oldest -= PROGRESS_MAX;
tick_cpu = state->progress_tick_misc[state->progress_ptr]
+ state->progress_tick_sched[state->progress_ptr]
+ state->progress_tick_raid[state->progress_ptr]
+ state->progress_tick_hash[state->progress_ptr];
tick_total = tick_cpu + state->progress_tick_io[state->progress_ptr];
oldest_tick_cpu = state->progress_tick_misc[oldest]
+ state->progress_tick_sched[oldest]
+ state->progress_tick_raid[oldest]
+ state->progress_tick_hash[oldest];
oldest_tick_total = oldest_tick_cpu + state->progress_tick_io[oldest];
delta_time = now - state->progress_time[oldest];
delta_pos = countpos - state->progress_pos[oldest];
delta_size = countsize - state->progress_size[oldest];
delta_tick_cpu = tick_cpu - oldest_tick_cpu;
delta_tick_total = tick_total - oldest_tick_total;
/* estimate the speed in MB/s */
if (delta_time != 0)
out_size_speed = (unsigned)(delta_size / MEGA / delta_time);
/* estimate the speed in block/s */
if (delta_time != 0)
out_block_speed = (unsigned)(delta_pos / delta_time);
/* estimate the cpu usage percentage */
if (delta_tick_total != 0)
out_cpu = muldiv(delta_tick_cpu, 100, delta_tick_total);
/* estimate the remaining time in minutes */
if (delta_pos != 0)
out_eta = muldiv(countmax - countpos, delta_time, delta_pos);
if (state->opt.force_stats) {
os_clear();
state_progress_graph(state, io, state->progress_ptr, oldest);
}
if (state->thermal_temperature_limit != 0) {
/* get the max temperature */
out_temperature = 0;
for (i = tommy_list_head(&state->thermallist); i != 0; i = i->next) {
struct snapraid_thermal* thermal = i->data;
if (out_temperature < thermal->latest_temperature) {
out_temperature = thermal->latest_temperature;
if (thermal->params.r_squared >= THERMAL_R_SQUARED_LIMIT)
out_steady = thermal->params.t_steady;
else
out_steady = 0;
}
}
} else {
out_temperature = 0;
out_steady = 0;
}
/* we have the output value */
out_computed = 1;
}
if (state->opt.gui) {
char str_eta[32];
char str_size_speed[32];
char str_cpu[32];
char str_temp[32];
char str_steady[32];
if (out_computed) {
snprintf(str_eta, sizeof(str_eta), "%u", out_eta);
snprintf(str_size_speed, sizeof(str_size_speed), "%u", out_size_speed);
snprintf(str_cpu, sizeof(str_cpu), "%u", out_cpu);
} else {
str_eta[0] = 0;
str_size_speed[0] = 0;
str_cpu[0] = 0;
}
if (out_temperature) {
snprintf(str_temp, sizeof(str_temp), "%u", out_temperature);
} else {
str_temp[0] = 0;
}
if (out_steady) {
snprintf(str_steady, sizeof(str_steady), "%u", out_steady);
} else {
str_steady[0] = 0;
}
log_tag("run:pos:%u:%u:%" PRIu64 ":%u:%s:%s:%s:%" PRIu64 ":%s:%s\n", blockpos, countpos, countsize, out_perc, str_eta, str_size_speed, str_cpu, (uint64_t)elapsed, str_temp, str_steady);
log_flush();
} else {
msg_bar("%u%%, %u MB", out_perc, (unsigned)(countsize / MEGA));
if (out_computed) {
msg_bar(", %u MB/s", out_size_speed);
msg_bar(", %u stripe/s", out_block_speed);
msg_bar(", CPU %u%%", out_cpu);
if (out_temperature != 0) {
if (out_steady != 0)
msg_bar(", Tmax %u (%u)", out_temperature, out_steady);
else
msg_bar(", Tmax %u", out_temperature);
}
unsigned out_eta_minutes = out_eta / 60; /* minutes */
msg_bar(", %u:%02u ETA", out_eta_minutes / 60, out_eta_minutes % 60);
}
msg_bar("\r");
msg_flush();
}
/* next position to fill */
++state->progress_ptr;
if (state->progress_ptr >= PROGRESS_MAX)
state->progress_ptr -= PROGRESS_MAX;
/* one more measure */
++state->progress_tick;
}
/* stop if requested */
if (global_interrupt) {
/* LCOV_EXCL_START */
if (!state->opt.gui) {
msg_bar("\n");
msg_flush();
}
log_fatal(EUSER, "Stopping for interruption at block %u\n", blockpos);
log_tag("sigint:%u: SIGINT received\n", blockpos);
log_flush();
return 1;
/* LCOV_EXCL_STOP */
}
return 0;
}
void state_usage_waste(struct snapraid_state* state)
{
uint64_t now = os_tick();
state->tick_last = now;
}
void state_usage_misc(struct snapraid_state* state)
{
uint64_t now = os_tick();
uint64_t delta = now - state->tick_last;
/* increment the time spent in computations */
state->tick_misc += delta;
state->tick_last = now;
}
void state_usage_sched(struct snapraid_state* state)
{
uint64_t now = os_tick();
uint64_t delta = now - state->tick_last;
/* increment the time spent in computations */
state->tick_sched += delta;
state->tick_last = now;
}
void state_usage_raid(struct snapraid_state* state)
{
uint64_t now = os_tick();
uint64_t delta = now - state->tick_last;
/* increment the time spent in computations */
state->tick_raid += delta;
state->tick_last = now;
}
void state_usage_hash(struct snapraid_state* state)
{
uint64_t now = os_tick();
uint64_t delta = now - state->tick_last;
/* increment the time spent in computations */
state->tick_hash += delta;
state->tick_last = now;
}
void state_usage_file(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
(void)state;
disk->progress_file = file;
}
void state_usage_disk(struct snapraid_state* state, struct snapraid_handle* handle_map, unsigned* waiting_map, unsigned waiting_mac)
{
uint64_t now = os_tick();
uint64_t delta = now - state->tick_last;
unsigned i;
/* increment the time spent in the data disks */
for (i = 0; i < waiting_mac; ++i) {
struct snapraid_disk* disk = handle_map[waiting_map[i]].disk;
if (!disk)
continue;
disk->tick += delta;
}
state->tick_io += delta;
state->tick_last = now;
}
void state_usage_parity(struct snapraid_state* state, unsigned* waiting_map, unsigned waiting_mac)
{
uint64_t now = os_tick();
uint64_t delta = now - state->tick_last;
unsigned i;
/* increment the time spent in the parity disk */
for (i = 0; i < waiting_mac; ++i)
state->parity[waiting_map[i]].tick += delta;
state->tick_io += delta;
state->tick_last = now;
}
void state_usage_print(struct snapraid_state* state)
{
/* set the latest data */
state_progress_latest(state);
if (msg_level < MSG_PROGRESS)
return;
/* print a graph for it */
state_progress_graph(state, 0, state->progress_ptr, PROGRESS_MAX);
}
void state_fscheck(struct snapraid_state* state, const char* ope)
{
tommy_node* i;
/* check the file-system on all disks */
for (i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
if (fs_check(disk) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency: File-system check for disk '%s' %s\n", disk->name, ope);
os_abort();
/* LCOV_EXCL_STOP */
}
}
}
void generate_configuration(const char* path)
{
struct snapraid_state state;
struct snapraid_content* content;
unsigned l, s;
tommy_node* j;
state_init(&state);
/* mark that we are without a configuration file */
state.no_conf = 1;
/* create the dummy content entry */
content = content_alloc(path, -1);
/* adds the content entry */
tommy_list_insert_tail(&state.contentlist, &content->node, content);
/* read the content file */
state_read(&state);
/* output a dummy configuration file */
printf("# Configuration file generated from %s\n", path);
printf("\n");
printf("# Use this blocksize\n");
printf("blocksize %u\n", state.block_size / KIBI);
printf("\n");
printf("# Use this hashsize\n");
printf("hashsize %u\n", BLOCK_HASH_SIZE);
printf("\n");
for (l = 0; l < state.level; ++l) {
printf("# Set the correct path for the %s files\n", lev_name(l));
printf("# You had %u of them:\n", state.parity[l].split_mac);
for (s = 0; s < state.parity[l].split_mac; ++s) {
printf("# %u:\n", s);
printf("# PATH:");
if (state.parity[l].split_map[s].path[0])
printf("%s", state.parity[l].split_map[s].path);
else
printf("?");
printf("\n");
printf("# SIZE:");
if (state.parity[l].split_map[s].size != PARITY_SIZE_INVALID)
printf("%" PRIu64, state.parity[l].split_map[s].size);
else
printf("?");
printf("\n");
printf("# UUID:");
if (state.parity[l].split_map[s].uuid[0])
printf("%s", state.parity[l].split_map[s].uuid);
else
printf("?");
printf("\n");
printf("#\n");
}
printf("%s ENTER_HERE_THE_PARITY_FILES_COMMA_SEPARATED\n", lev_config_name(l));
printf("\n");
}
printf("# Add any other content file\n");
printf("content %s\n", path);
printf("\n");
for (j = state.maplist; j; j = j->next) {
struct snapraid_map* map = j->data;
struct snapraid_disk* disk;
printf("# Set the correct dir for disk '%s'\n", map->name);
if (map->uuid[0])
printf("# Disk '%s' is the one with id '%s'\n", map->name, map->uuid);
disk = find_disk_by_name(&state, map->name);
if (disk && disk->filelist) {
struct snapraid_file* file = disk->filelist->data;
if (file) {
printf("# and containing: %s\n", fmt_poll(disk, file->sub));
}
}
printf("data %s ENTER_HERE_THE_DIR\n", map->name);
printf("\n");
}
state_done(&state);
}
/**
* Flush and sync the parity files.
*/
int state_flush(struct snapraid_state* state, struct snapraid_io* io, struct snapraid_parity_handle* parity_handle, block_off_t blockcur)
{
unsigned l;
if (io)
io_flush(io);
/* flush all parity handles to ensure data is written to disk */
for (l = 0; l < state->level; ++l) {
int ret = parity_sync(&parity_handle[l]);
if (ret == -1) {
/* LCOV_EXCL_START */
log_tag("parity_error_io:%u:%s: Sync error. %s.\n", blockcur, lev_config_name(l), strerror(errno));
log_fatal(errno, "DANGER! Unexpected input/output error in disk %s. It isn't possible to continue.\n", lev_config_name(l));
return -1;
/* LCOV_EXCL_STOP */
}
}
return 0;
}
void state_load_ignore_file(tommy_list* filter_list, const char* path, const char* sub)
{
STREAM* f;
int line;
f = sopen_read(path, 0);
if (!f) {
/* LCOV_EXCL_START */
log_error(errno, "Error opening the ignore file '%s'. %s.\n", path, strerror(errno));
return;
/* LCOV_EXCL_STOP */
}
line = 1;
while (1) {
char buffer[PATH_MAX];
int ret;
int c;
/* skip initial spaces */
sgetspace(f);
/* get the whole line */
ret = sgetline(f, buffer, sizeof(buffer));
if (ret < 0) {
/* LCOV_EXCL_START */
log_error(EUSER, "Too long line in '%s' at line %u\n", path, line);
break;
/* LCOV_EXCL_STOP */
}
if (buffer[0] == 0) {
/* allow empty lines */
} else if (buffer[0] == '#') {
/* ignore comment lines */
} else {
struct snapraid_filter* filter;
filter = filter_alloc_file(-1, sub, buffer);
if (!filter) {
/* LCOV_EXCL_START */
log_error(EUSER, "Invalid ignore specification '%s' in '%s' at line %u\n", buffer, path, line);
break;
/* LCOV_EXCL_STOP */
}
tommy_list_insert_tail(filter_list, &filter->node, filter);
}
/* next line */
c = sgeteol(f);
if (c == EOF) {
break;
}
if (c != '\n') {
/* LCOV_EXCL_START */
log_error(EUSER, "Extra data in '%s' at line %u\n", path, line);
break;
/* LCOV_EXCL_STOP */
}
++line;
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_error(errno, "Error reading the ignore file '%s' at line %u\n", path, line);
/* LCOV_EXCL_STOP */
}
sclose(f);
}
#define SNAPSHOT_CONTAINER ".snapraid"
#define SNAPSHOT_PENDING "pending"
#define SNAPSHOT_STABLE "stable"
static int state_snapshot_dir(const char* dir, struct snapraid_disk* disk)
{
struct stat st;
if (lstat(dir, &st) != 0)
return -1;
if (!S_ISDIR(st.st_mode))
return -1;
if (disk) {
pathcpy(disk->dir, sizeof(disk->dir), dir);
disk->dir_device = st.st_dev;
}
return 0;
}
int state_snapshot_new(struct snapraid_state* state)
{
if (!state->snapshot)
return 0;
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
char root[PATH_MAX];
char container[PATH_MAX];
char vol[PATH_MAX];
int ret;
/* check if it supports snapshot */
if (fssnapshot(disk->mount_point, root, sizeof(root)) != 0)
continue;
size_t root_len = strlen(root);
pathcpy(container, sizeof(container), root);
pathcat(container, sizeof(container), SNAPSHOT_CONTAINER);
/* create the snapshot container directory if not existing */
ret = mkdir(container, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH);
if (ret != 0 && errno != EEXIST) {
log_error(errno, "Error creating '%s'. %s.\n", container, strerror(errno));
return -1;
}
/* delete a potential previous pending snapshot */
if (fssnapshot_delete(container, SNAPSHOT_PENDING) != 0) {
log_fatal(errno, "Failed to delete pending snapshot in '%s'. %s.\n", container, strerror(errno));
return -1;
}
/* create a new snapshot */
if (fssnapshot_create(root, container, SNAPSHOT_PENDING) != 0) {
log_fatal(errno, "Failed to create pending snapshot '%s'. %s.\n", container, strerror(errno));
return -1;
}
msg_progress("Created disk %s pending snapshot...\n", disk->name);
/* setup the snapshot in use */
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_PENDING "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
if (state_snapshot_dir(vol, disk) != 0) {
log_error(errno, "Error stating snapshot dir '%s'. %s.\n", vol, strerror(errno));
return -1;
}
/* store the snapshot root */
pathcpy(disk->snapshot_root, sizeof(disk->snapshot_root), root);
}
return 0;
}
int state_snapshot_commit(struct snapraid_state* state)
{
if (!state->snapshot)
return 0;
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
char container[PATH_MAX];
/* check if it supports snapshot */
if (disk->snapshot_root[0] == 0)
continue;
pathcpy(container, sizeof(container), disk->snapshot_root);
pathcat(container, sizeof(container), SNAPSHOT_CONTAINER);
/* delete a potential previous stable snapshot */
if (fssnapshot_delete(container, SNAPSHOT_STABLE) != 0) {
log_fatal(errno, "Failed to delete stable snapshot in '%s'. %s.\n", container, strerror(errno));
return -1;
}
/* rename pending to stable */
if (fssnapshot_rename(container, SNAPSHOT_PENDING, SNAPSHOT_STABLE) != 0) {
log_fatal(errno, "Failed to rename snapshot in '%s'. %s.\n", container, strerror(errno));
return -1;
}
msg_progress("Committed disk %s stable snapshot...\n", disk->name);
}
return 0;
}
void state_snapshot_read(struct snapraid_state* state)
{
if (!state->snapshot)
return;
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
char root[PATH_MAX];
char container[PATH_MAX];
char vol[PATH_MAX];
/* check if it supports snapshot */
if (fssnapshot(disk->mount_point, root, sizeof(root)) != 0)
continue;
size_t root_len = strlen(root);
pathcpy(container, sizeof(container), root);
pathcat(container, sizeof(container), SNAPSHOT_CONTAINER);
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_PENDING "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
if (state_snapshot_dir(vol, disk) == 0) {
msg_progress("Using disk %s pending snapshot...\n", disk->name);
} else {
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
if (state_snapshot_dir(vol, disk) == 0) {
msg_progress("Using disk %s stable snapshot...\n", disk->name);
} else {
/* fallback to standard mount point */
msg_progress("Using disk %s live filesystem...\n", disk->name);
}
}
}
}
void state_snapshot_write(struct snapraid_state* state, tommy_list* filterlist_disk)
{
if (!state->snapshot)
return;
/* if the filter disk is empty, all disks are potentially written */
if (tommy_list_empty(filterlist_disk)) {
msg_progress("Using live filesystems...\n");
return;
}
for (tommy_node* i = state->disklist; i != 0; i = i->next) {
struct snapraid_disk* disk = i->data;
char root[PATH_MAX];
char container[PATH_MAX];
char vol[PATH_MAX];
/* check if it supports snapshot */
if (fssnapshot(disk->mount_point, root, sizeof(root)) != 0)
continue;
size_t root_len = strlen(root);
/* if the disk is filtered in, it could be written, then we cannot use the snapshot */
if (filter_path(filterlist_disk, 0, disk->name, 0) == 0) {
msg_progress("Using disk %s live filesystem...\n", disk->name);
continue;
}
pathcpy(container, sizeof(container), root);
pathcat(container, sizeof(container), SNAPSHOT_CONTAINER);
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_PENDING "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
if (state_snapshot_dir(vol, disk) == 0) {
msg_progress("Using disk %s pending snapshot...\n", disk->name);
/* if there is a dealloc list */
if (!tommy_list_empty(&disk->dealloclist)) {
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
/* if there is a previous snapshot */
if (state_snapshot_dir(vol, 0) == 0) {
msg_progress("Importing disk %s stable snapshot %" PRIu64 " deallocated files...\n", disk->name, tommy_list_count(&disk->dealloclist));
state_dealloc(state, vol, &disk->dealloclist);
}
}
} else {
pathcpy(vol, sizeof(vol), container);
pathcat(vol, sizeof(vol), "/" SNAPSHOT_STABLE "/");
pathcat(vol, sizeof(vol), disk->mount_point + root_len);
if (state_snapshot_dir(vol, disk) == 0) {
msg_progress("Using disk %s stable snapshot...\n", disk->name);
} else {
/* fallback to standard mount point */
msg_progress("Using disk %s live filesystem...\n", disk->name);
}
}
}
}
Reference in a new issue View git blame Copy permalink