rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Stores the free space in the content file.

Browse source 
This avoids the need to access data and parity disks in
the status command.
Also removed some data disk access, in list, dup and similar
commands that don't need to scan the disks.
This commit is contained in:
Andrea Mazzoleni 2014-11-09 19:14:29 +01:00
parent 8ee1131bed
commit 7e9a5e6a6d
12 changed files with 519 additions and 128 deletions
Download patch file Download diff file Expand all files Collapse all files

8
cmdline/elem.c
View file

@ -649,6 +649,8 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
disk->device = dev;
disk->tick = 0;
disk->total_blocks = 0;
disk->free_blocks = 0;
disk->first_free_block = 0;
disk->has_volatile_inodes = 0;
disk->has_unreliable_physical = 0;
@ -727,14 +729,16 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax)
return 1;
}
struct snapraid_map* map_alloc(const char* name, unsigned position, const char* uuid)
struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid)
{
struct snapraid_map* map;
map = malloc_nofail(sizeof(struct snapraid_map));
pathcpy(map->name, sizeof(map->name), name);
pathcpy(map->uuid, sizeof(map->uuid), uuid);
map->position = position;
map->total_blocks = total_blocks;
map->free_blocks = free_blocks;
pathcpy(map->uuid, sizeof(map->uuid), uuid);
return map;
}

17
cmdline/elem.h
View file

@ -308,6 +308,9 @@ struct snapraid_disk {
uint64_t tick; /**< Usage time of the disk. */
block_off_t total_blocks; /**< Number of total blocks. */
block_off_t free_blocks; /**< Number of free blocks at the last sync. */
/**
* First free searching block.
* Note that it doesn't necessarely point at the first free block,
@ -364,8 +367,10 @@ struct snapraid_disk {
*/
struct snapraid_map {
char name[PATH_MAX]; /**< Name of the disk. */
unsigned position; /**< Position of the disk in the parity. */
char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
block_off_t total_blocks; /**< Number of total blocks. */
block_off_t free_blocks; /**< Number of free blocks at last 'sync'. */
unsigned position; /**< Position of the disk in the parity. */
/* nodes for data structures */
tommy_node node;
@ -376,7 +381,15 @@ struct snapraid_map {
*/
struct snapraid_parity {
char path[PATH_MAX]; /**< Path of the parity file. */
char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */
uint64_t device; /**< Device identifier of the parity. */
block_off_t total_blocks; /**< Number of total blocks. */
block_off_t free_blocks; /**< Number of free blocks at the last sync. */
/**
* Cumulative time used for parity disks.
*/
uint64_t tick;
};
/**
@ -869,7 +882,7 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
* Allocates a disk mapping.
* Uses uuid="" if not available.
*/
struct snapraid_map* map_alloc(const char* name, unsigned position, const char* uuid);
struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid);
/**
* Deallocates a disk mapping.

12
cmdline/mingw.c
View file

@ -1431,13 +1431,14 @@ int filephy(const char* file, uint64_t size, uint64_t* physical)
return 0;
}
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space)
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space)
{
/* all FAT/exFAT/NTFS when managed from Windows have persistent inodes */
if (has_persistent_inode)
*has_persistent_inode = 1;
if (free_space) {
if (free_space || total_space) {
ULARGE_INTEGER total_bytes;
ULARGE_INTEGER total_free_bytes;
DWORD attr;
char dir[PATH_MAX];
@ -1484,12 +1485,15 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space)
/* get the free space of the directory */
/* note that it must be a directory */
if (!GetDiskFreeSpaceExW(convert(dir), 0, 0, &total_free_bytes)) {
if (!GetDiskFreeSpaceExW(convert(dir), 0, &total_bytes, &total_free_bytes)) {
windows_errno(GetLastError());
return -1;
}
*free_space = total_free_bytes.QuadPart;
if (total_space)
*total_space = total_bytes.QuadPart;
if (free_space)
*free_space = total_free_bytes.QuadPart;
}
return 0;

2
cmdline/portable.h
View file

@ -243,7 +243,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical);
* Checks if the underline filesystem support persistent inodes.
* Returns -1 on error, 0 on success.
*/
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space);
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space);
/**
* Get the tick counter value.

2
cmdline/scan.c
View file

@ -1384,7 +1384,7 @@ void state_scan(struct snapraid_state* state, int is_diff)
printf("Scanning disk %s...\n", disk->name);
/* check if the disk supports persistent inodes */
ret = fsinfo(disk->dir, &has_persistent_inode, 0);
ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno));

3
cmdline/scrub.c
View file

@ -629,6 +629,9 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
printf("Scrubbing...\n");
/* after opening parity files, refresh the free info */
state_refresh(state);
error = 0;
ret = state_scrub_process(state, parity_ptr, 0, blockmax, timelimit, lastlimit, now);

19
cmdline/snapraid.c
View file

@ -237,6 +237,7 @@ void config(char* conf, size_t conf_size, const char* argv0)
#define OPT_TEST_FORCE_SCAN_WINFIND 279
#define OPT_TEST_IMPORT_CONTENT 280
#define OPT_TEST_FORCE_PROGRESS 281
#define OPT_TEST_SKIP_DISK_ACCESS 282
#if HAVE_GETOPT_LONG
struct option long_options[] = {
@ -341,6 +342,9 @@ struct option long_options[] = {
/* Force immediate progress state update */
{ "test-force-progress", 0, 0, OPT_TEST_FORCE_PROGRESS },
/* Skip the disk access */
{ "test-skip-disk-access", 0, 0, OPT_TEST_SKIP_DISK_ACCESS },
{ 0, 0, 0, 0 }
};
#endif
@ -604,6 +608,9 @@ int main(int argc, char* argv[])
case OPT_TEST_SKIP_PARITY_ACCESS :
opt.skip_parity_access = 1;
break;
case OPT_TEST_SKIP_DISK_ACCESS :
opt.skip_disk_access = 1;
break;
case OPT_TEST_FORCE_MURMUR3 :
opt.force_murmur3 = 1;
break;
@ -828,6 +835,17 @@ int main(int argc, char* argv[])
}
}
switch (operation) {
case OPERATION_LIST :
case OPERATION_DUP :
case OPERATION_STATUS :
case OPERATION_REWRITE :
case OPERATION_REHASH :
/* avoid to check and access data disks if not needed */
opt.skip_disk_access = 1;
break;
}
switch (operation) {
case OPERATION_DIFF :
case OPERATION_LIST :
@ -835,6 +853,7 @@ int main(int argc, char* argv[])
case OPERATION_POOL :
case OPERATION_STATUS :
case OPERATION_REWRITE :
case OPERATION_REHASH :
case OPERATION_NANO :
/* avoid to check and access parity disks if not needed */
opt.skip_parity_access = 1;

520
cmdline/state.c
View file

@ -85,8 +85,11 @@ void state_init(struct snapraid_state* state)
state->file_mode = MODE_SEQUENTIAL;
for (l = 0; l < LEV_MAX; ++l) {
state->parity[l].path[0] = 0;
state->parity[l].uuid[0] = 0;
state->parity[l].device = 0;
state->tick_parity[l] = 0;
state->parity[l].total_blocks = 0;
state->parity[l].free_blocks = 0;
state->parity[l].tick = 0;
}
state->tick_io = 0;
state->tick_cpu = 0;
@ -173,8 +176,8 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
}
/* check if all the data and parity disks are different */
if (!state->opt.skip_device) {
/* check device of data disks */
if (!state->opt.skip_device && !state->opt.skip_disk_access) {
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
@ -212,17 +215,19 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
}
for (l = 0; l < state->level; ++l) {
if (disk->device == state->parity[l].device) {
/* LCOV_EXCL_START */
fprintf(stderr, "Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].path);
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
if (disk->device == state->parity[l].device) {
/* LCOV_EXCL_START */
fprintf(stderr, "Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].path);
#ifdef _WIN32
fprintf(stderr, "Both have the serial number '%" PRIx64 "'.\n", disk->device);
fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
fprintf(stderr, "to change one of the disk serial.\n");
fprintf(stderr, "Both have the serial number '%" PRIx64 "'.\n", disk->device);
fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n");
fprintf(stderr, "to change one of the disk serial.\n");
#endif
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
}
@ -238,7 +243,10 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
/* LCOV_EXCL_STOP */
}
}
}
/* check device of parity disks */
if (!state->opt.skip_device && !state->opt.skip_parity_access) {
#ifdef _WIN32
for (l = 0; l < state->level; ++l) {
if (state->parity[l].device == 0) {
@ -251,16 +259,6 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
/* LCOV_EXCL_STOP */
}
}
if (state->pool[0] != 0 && state->pool_device == 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Disk '%s' has a zero serial number.\n", state->pool);
fprintf(stderr, "This is not necessarely wrong, but for using SnapRAID\n");
fprintf(stderr, "it's better to change the serial number of the disk.\n");
fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
#endif
for (l = 0; l < state->level; ++l) {
@ -295,6 +293,21 @@ static void state_config_check(struct snapraid_state* state, const char* path, t
}
}
/* check device of pool disk */
#ifdef _WIN32
if (!state->opt.skip_device) {
if (state->pool[0] != 0 && state->pool_device == 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Disk '%s' has a zero serial number.\n", state->pool);
fprintf(stderr, "This is not necessarely wrong, but for using SnapRAID\n");
fprintf(stderr, "it's better to change the serial number of the disk.\n");
fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
#endif
/* count the content files */
if (!state->opt.skip_device) {
unsigned content_count;
@ -529,7 +542,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
state->parity[level].device = st.st_dev;
} else {
/* if parity is skipped, uses a fake device */
state->parity[level].device = -1LL - level;
state->parity[level].device = 0;
}
/* adjust the level */
@ -662,8 +675,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
} else if (strcmp(tag, "disk") == 0) {
char dir[PATH_MAX];
char device[PATH_MAX];
struct stat st;
struct snapraid_disk* disk;
uint64_t dev;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
@ -699,12 +712,6 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* get the device of the dir */
pathimport(device, sizeof(device), dir);
if (stat(device, &st) != 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check if the disk name already exists */
for (i = state->disklist; i != 0; i = i->next) {
@ -719,7 +726,23 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* LCOV_EXCL_STOP */
}
disk = disk_alloc(buffer, dir, st.st_dev);
if (!state->opt.skip_disk_access) {
struct stat st;
if (stat(device, &st) != 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
dev = st.st_dev;
} else {
/* if data disks are skipped, uses a fake device */
dev = 0;
}
disk = disk_alloc(buffer, dir, dev);
tommy_list_insert_tail(&state->disklist, &disk->node, disk);
} else if (strcmp(tag, "nohidden") == 0) {
@ -930,6 +953,7 @@ static void state_map(struct snapraid_state* state)
tommy_node* i;
unsigned uuid_mismatch;
unsigned diskcount;
unsigned l;
/* removes all the mapping without a disk */
/* this happens when a disk is removed from the configuration file */
@ -967,8 +991,12 @@ static void state_map(struct snapraid_state* state)
break;
}
}
if (j != 0)
if (j != 0) {
/* mapping is present, then copy the free blocks into to disk */
disk->total_blocks = map->total_blocks;
disk->free_blocks = map->free_blocks;
continue;
}
/* mapping not found, search for an hole */
while (1) {
@ -989,7 +1017,7 @@ static void state_map(struct snapraid_state* state)
}
/* insert the new mapping */
map = map_alloc(disk->name, hole, "");
map = map_alloc(disk->name, hole, 0, 0, "");
tommy_list_insert_tail(&state->maplist, &map->node, map);
}
@ -998,50 +1026,73 @@ static void state_map(struct snapraid_state* state)
if (state->no_conf)
return;
/* checks if mapping match the disk uuid */
uuid_mismatch = 0;
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
char uuid[UUID_MAX];
int ret;
disk = find_disk(state, map->name);
if (disk == 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* checks if mapping match the disk uuid */
if (!state->opt.skip_disk_access) {
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
char uuid[UUID_MAX];
int ret;
ret = devuuid(disk->device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore but marks the disk with unsupported UUID */
disk->has_unsupported_uuid = 1;
continue;
}
/* if the uuid is changed */
if (strcmp(uuid, map->uuid) != 0) {
/* mark the disk as with an UUID change */
disk->has_different_uuid = 1;
/* if the previous uuid is available */
if (map->uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
fprintf(stderr, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, uuid);
} else {
/* no message here, because having a disk without */
/* UUID is the normal state of an empty disk */
disk->had_empty_uuid = 1;
disk = find_disk(state, map->name);
if (disk == 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* update the uuid in the mapping, */
pathcpy(map->uuid, sizeof(map->uuid), uuid);
ret = devuuid(disk->device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore but marks the disk with unsupported UUID */
disk->has_unsupported_uuid = 1;
continue;
}
/* write the new state with the new uuid */
state->need_write = 1;
/* if the uuid is changed */
if (strcmp(uuid, map->uuid) != 0) {
/* mark the disk as with an UUID change */
disk->has_different_uuid = 1;
/* if the previous uuid is available */
if (map->uuid[0] != 0) {
/* count the number of uuid change */
++uuid_mismatch;
fprintf(stderr, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, uuid);
} else {
/* no message here, because having a disk without */
/* UUID is the normal state of an empty disk */
disk->had_empty_uuid = 1;
}
/* update the uuid in the mapping, */
pathcpy(map->uuid, sizeof(map->uuid), uuid);
/* write the new state with the new uuid */
state->need_write = 1;
}
}
}
/* checks the parity uuid */
if (!state->opt.skip_parity_access) {
for (l = 0; l < state->level; ++l) {
char uuid[UUID_MAX];
int ret;
ret = devuuid(state->parity[l].device, uuid, sizeof(uuid));
if (ret != 0) {
/* uuid not available, just ignore */
continue;
}
/* update the uuid in the mapping */
/* at now we don't enforce uuid for parity */
/* as the user is expected to have a different file name */
/* for each parity file */
pathcpy(state->parity[l].uuid, sizeof(state->parity[l].uuid), uuid);
}
}
@ -1085,6 +1136,67 @@ static void state_map(struct snapraid_state* state)
}
}
void state_refresh(struct snapraid_state* state)
{
tommy_node* i;
unsigned l;
/* for all disks */
for (i = state->maplist; i != 0; i = i->next) {
struct snapraid_map* map = i->data;
struct snapraid_disk* disk;
uint64_t total_space;
uint64_t free_space;
int ret;
disk = find_disk(state, map->name);
if (disk == 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = fsinfo(disk->dir, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* set the new free blocks */
map->total_blocks = total_space / state->block_size;
map->free_blocks = free_space / state->block_size;
/* also update the disk info */
disk->total_blocks = map->total_blocks;
disk->free_blocks = map->free_blocks;
}
/* for all parities */
for (l = 0; l < state->level; ++l) {
uint64_t total_space;
uint64_t free_space;
int ret;
ret = fsinfo(state->parity[l].path, 0, &total_space, &free_space);
if (ret != 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing file '%s' to get filesystem info. %s.\n", state->parity[l].path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* set the new free blocks */
state->parity[l].total_blocks = total_space / state->block_size;
state->parity[l].free_blocks = free_space / state->block_size;
}
/* note what we don't set need_write = 1, because we don't want */
/* to update the content file only for the free space info. */
}
/**
* Checks the content.
*/
@ -2020,10 +2132,12 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else if (strcmp(tag, "map") == 0) {
} else if (strcmp(tag, "map") == 0 || strcmp(tag, "mapping") == 0) {
struct snapraid_map* map;
char uuid[UUID_MAX];
uint32_t v_pos;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
@ -2049,6 +2163,46 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
/* LCOV_EXCL_STOP */
}
/* from SnapRAID 7.0 the 'mapping' command includes the free space */
if (strcmp(tag, "mapping") == 0) {
c = sgetc(f);
if (c != ' ') {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetu32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
c = sgetc(f);
if (c != ' ') {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetu32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else {
v_total_blocks = 0;
v_free_blocks = 0;
}
/* check if the uuid is present */
/* uuid must be the latest one as it could be also empty */
c = sgetc(f);
if (c != ' ') {
sungetc(c, f);
@ -2064,9 +2218,80 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
}
}
map = map_alloc(buffer, v_pos, uuid);
map = map_alloc(buffer, v_pos, v_total_blocks, v_free_blocks, uuid);
tommy_list_insert_tail(&state->maplist, &map->node, map);
} else if (strcmp(tag, "parity") == 0) {
/* from SnapRAID 7.0 the 'parity' command includes the free space */
char uuid[UUID_MAX];
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
ret = sgetu32(f, &v_level);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
c = sgetc(f);
if (c != ' ') {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetu32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
c = sgetc(f);
if (c != ' ') {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetu32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check if the uuid is present */
/* uuid must be the latest one as it could be also empty */
c = sgetc(f);
if (c != ' ') {
sungetc(c, f);
uuid[0] = 0;
} else {
/* read the uuid */
ret = sgettok(f, uuid, sizeof(uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid);
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
} else if (strcmp(tag, "sign") == 0) {
uint32_t sign;
@ -2174,6 +2399,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
block_off_t b;
block_off_t blockmax;
int info_has_rehash;
unsigned l;
count_file = 0;
count_hardlink = 0;
@ -2277,16 +2503,21 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
/* save the mapping only for not empty disks */
if (!disk_is_empty(disk, blockmax)) {
sputsl("map ", f);
sputsl("mapping ", f);
sputs(map->name, f);
sputc(' ', f);
sputu32(map->position, f);
sputc(' ', f);
sputu32(map->total_blocks, f);
sputc(' ', f);
sputu32(map->free_blocks, f);
/* if there is an uuid, print it */
if (map->uuid[0]) {
sputc(' ', f);
sputs(map->uuid, f);
}
sputeol(f);
if (serror(f)) {
/* LCOV_EXCL_START */
@ -2303,6 +2534,30 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
}
}
/* for each parity */
for (l = 0; l < state->level; ++l) {
sputsl("parity ", f);
sputu32(l, f);
sputc(' ', f);
sputu32(state->parity[l].total_blocks, f);
sputc(' ', f);
sputu32(state->parity[l].free_blocks, f);
/* if there is an uuid, print it */
if (state->parity[l].uuid[0]) {
sputc(' ', f);
sputs(state->parity[l].uuid, f);
}
sputeol(f);
if (serror(f)) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
@ -2625,7 +2880,16 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0) {
/*
* File format versions:
* - SNAPCNT1/SnapRAID 4.0 First version.
* - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
* The previous 'm' entry is now deprecated, but supported for importing.
* Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
*/
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0) {
/* LCOV_EXCL_START */
if (memcmp(buffer, "SNAPCNT", 7) != 0) {
decoding_error(path, f);
@ -3335,10 +3599,12 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else if (c == 'm') {
} else if (c == 'm' || c == 'M') {
struct snapraid_map* map;
char uuid[UUID_MAX];
uint32_t v_pos;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
struct snapraid_disk* disk;
ret = sgetbs(f, buffer, sizeof(buffer));
@ -3357,6 +3623,28 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* LCOV_EXCL_STOP */
}
/* from SnapRAID 7.0 the 'M' command includes the free space */
if (c == 'M') {
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else {
v_total_blocks = 0;
v_free_blocks = 0;
}
/* read the uuid */
ret = sgetbs(f, uuid, sizeof(uuid));
if (ret < 0) {
@ -3366,7 +3654,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
/* LCOV_EXCL_STOP */
}
map = map_alloc(buffer, v_pos, uuid);
map = map_alloc(buffer, v_pos, v_total_blocks, v_free_blocks, uuid);
tommy_list_insert_tail(&state->maplist, &map->node, map);
@ -3385,6 +3673,52 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
tommy_array_grow(&disk_mapping, mapping_max + 1);
tommy_array_set(&disk_mapping, mapping_max, disk);
++mapping_max;
} else if (c == 'P') {
/* from SnapRAID 7.0 the 'P' command includes the free space */
char uuid[UUID_MAX];
uint32_t v_level;
uint32_t v_total_blocks;
uint32_t v_free_blocks;
ret = sgetb32(f, &v_level);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_total_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetb32(f, &v_free_blocks);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
ret = sgetbs(f, uuid, sizeof(uuid));
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* if we use this parity entry */
if (v_level < state->level) {
/* set the parity info */
pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid);
state->parity[v_level].total_blocks = v_total_blocks;
state->parity[v_level].free_blocks = v_free_blocks;
}
} else if (c == 'N') {
uint32_t crc_stored;
uint32_t crc_computed;
@ -3475,6 +3809,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
time_t info_oldest;
int info_has_rehash;
int mapping_idx;
unsigned l;
count_file = 0;
count_hardlink = 0;
@ -3513,7 +3848,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
}
/* write header */
swrite("SNAPCNT1\n\3\0\0", 12, f);
swrite("SNAPCNT2\n\3\0\0", 12, f);
/* write block size and block max */
sputc('z', f);
@ -3588,9 +3923,11 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
/* save the mapping only for not empty disks */
if (!disk_is_empty(disk, blockmax)) {
sputc('m', f);
sputc('M', f);
sputbs(map->name, f);
sputb32(map->position, f);
sputb32(map->total_blocks, f);
sputb32(map->free_blocks, f);
sputbs(map->uuid, f);
if (serror(f)) {
/* LCOV_EXCL_START */
@ -3608,6 +3945,21 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
}
}
/* for each parity */
for (l = 0; l < state->level; ++l) {
sputc('P', f);
sputb32(l, f);
sputb32(state->parity[l].total_blocks, f);
sputb32(state->parity[l].free_blocks, f);
sputbs(state->parity[l].uuid, f);
if (serror(f)) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
}
/* for each disk */
for (i = state->disklist; i != 0; i = i->next) {
tommy_node* j;
@ -4443,9 +4795,9 @@ void state_usage_print(struct snapraid_state* state)
tick_max = disk->tick;
}
for (l = 0; l < state->level; ++l) {
tick_total += state->tick_parity[l];
if (state->tick_parity[l] > tick_max)
tick_max = state->tick_parity[l];
tick_total += state->parity[l].tick;
if (state->parity[l].tick > tick_max)
tick_max = state->parity[l].tick;
}
if (!tick_total)
@ -4464,11 +4816,11 @@ void state_usage_print(struct snapraid_state* state)
printf("Time for parity:");
for (l = 0; l < state->level; ++l) {
if (state->tick_parity[l] == tick_max)
if (state->parity[l].tick == tick_max)
printf(" %s:", lev_config_name(l));
else
printf(" ");
printf("%" PRIu64 "%%", state->tick_parity[l] * 100U / tick_total);
printf("%" PRIu64 "%%", state->parity[l].tick * 100U / tick_total);
}
printf("\n");
}

15
cmdline/state.h
View file

@ -82,6 +82,7 @@ struct snapraid_option {
int skip_self; /**< Skip the self-test. */
int skip_content_check; /**< Relax some content file checks. */
int skip_parity_access; /**< Skip the parity access for commands that don't need it. */
int skip_disk_access; /**< Skip the data disk access for commands that don't need it. */
int kill_after_sync; /**< Kill the process after sync without saving the final state. */
int force_murmur3; /**< Force Murmur3 choice. */
int force_spooky2; /**< Force Spooky2 choice. */
@ -140,11 +141,6 @@ struct snapraid_state {
*/
uint64_t tick_io;
/**
* Cumulative time used for parity disks.
*/
uint64_t tick_parity[LEV_MAX];
/**
* Last time used for time measure.
*/
@ -263,6 +259,13 @@ void state_list(struct snapraid_state* state);
*/
void state_pool(struct snapraid_state* state);
/**
* Refresh the free space info.
*
* Note that it requires disks access.
*/
void state_refresh(struct snapraid_state* state);
/**
* Filter files, symlinks and dirs.
* Apply an additional filter to the list currently loaded.
@ -344,7 +347,7 @@ static inline void state_usage_parity(struct snapraid_state* state, unsigned lev
assert(level < LEV_MAX);
/* increment the time spent in the parity disk */
state->tick_parity[level] += delta;
state->parity[level].tick += delta;
state->tick_io += delta;
state->tick_last = now;

40
cmdline/status.c
View file

@ -52,6 +52,7 @@ int state_status(struct snapraid_state* state)
block_off_t bad;
block_off_t rehash;
block_off_t count;
unsigned l;
unsigned dayoldest, daymedian, daynewest;
unsigned bar[GRAPH_COLUMN];
unsigned barpos;
@ -68,9 +69,7 @@ int state_status(struct snapraid_state* state)
uint64_t base = 1024 * 1024 * (uint64_t)1024;
block_off_t parity_block_free;
unsigned unsynced_blocks;
uint64_t free_space;
uint64_t all_wasted;
int ret;
/* get the present time */
now = time(0);
@ -80,18 +79,15 @@ int state_status(struct snapraid_state* state)
fprintf(stdlog, "summary:block_size:%u\n", state->block_size);
fprintf(stdlog, "summary:parity_block_count:%u\n", blockmax);
/* get the free space on the first parity disk */
ret = fsinfo(state->parity[0].path, 0, &free_space);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing file '%s' to get filesystem info. %s.\n", state->parity[0].path, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
/* get the minimum parity free space */
parity_block_free = state->parity[0].free_blocks;
for (l = 0; l < state->level; ++l) {
fprintf(stdlog, "summary:parity_block_total:%s:%u\n", lev_config_name(l), state->parity[l].total_blocks);
fprintf(stdlog, "summary:parity_block_free:%s:%u\n", lev_config_name(l), state->parity[l].free_blocks);
if (state->parity[l].free_blocks < parity_block_free)
parity_block_free = state->parity[l].free_blocks;
}
parity_block_free = free_space / state->block_size;
fprintf(stdlog, "summary:parity_block_free:%s:%u\n", lev_config_name(0), parity_block_free);
fprintf(stdlog, "summary:parity_block_free_min:%u\n", parity_block_free);
printf("\n");
printf(" Files Fragmented Excess Wasted Used Free Use Name\n");
@ -115,7 +111,6 @@ int state_status(struct snapraid_state* state)
block_off_t disk_block_count = 0;
uint64_t disk_file_size = 0;
block_off_t disk_block_latest_used = 0;
block_off_t disk_block_free;
block_off_t disk_block_max_by_space;
block_off_t disk_block_max_by_parity;
block_off_t disk_block_max;
@ -167,18 +162,8 @@ int state_status(struct snapraid_state* state)
disk_file_size += file->size;
}
/* get the free space on the disk */
ret = fsinfo(disk->dir, 0, &free_space);
if (ret < 0) {
/* LCOV_EXCL_START */
fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno));
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
disk_block_free = free_space / state->block_size;
disk_block_max_by_space = disk_block_count + disk_block_free;
/* get the free block info */
disk_block_max_by_space = disk_block_count + disk->free_blocks;
disk_block_max_by_parity = blockmax + parity_block_free;
/* the maximum usable space in a disk is limited by the smaller */
@ -213,7 +198,8 @@ int state_status(struct snapraid_state* state)
fprintf(stdlog, "summary:disk_excess_fragment_count:%s:%u\n", disk->name, disk_extra_fragment);
fprintf(stdlog, "summary:disk_file_size:%s:%" PRIu64 "\n", disk->name, disk_file_size);
fprintf(stdlog, "summary:disk_block_allocated:%s:%u\n", disk->name, disk_block_latest_used + 1);
fprintf(stdlog, "summary:disk_block_free:%s:%u\n", disk->name, disk_block_free);
fprintf(stdlog, "summary:disk_block_total:%s:%u\n", disk->name, disk->total_blocks);
fprintf(stdlog, "summary:disk_block_free:%s:%u\n", disk->name, disk->free_blocks);
fprintf(stdlog, "summary:disk_block_max_by_space:%s:%u\n", disk->name, disk_block_max_by_space);
fprintf(stdlog, "summary:disk_block_max_by_parity:%s:%u\n", disk->name, disk_block_max_by_parity);
fprintf(stdlog, "summary:disk_block_max:%s:%u\n", disk->name, disk_block_max);

3
cmdline/sync.c
View file

@ -1090,6 +1090,9 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
}
}
/* after opening and resizing parity files, refresh the free info */
state_refresh(state);
unrecoverable_error = 0;
if (state->opt.prehash) {

6
cmdline/unix.c
View file

@ -205,7 +205,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical)
return 0;
}
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space)
int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space)
{
#if HAVE_STATFS && HAVE_STRUCT_STATFS_F_TYPE
struct statfs st;
@ -249,6 +249,8 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space)
break;
}
if (total_space)
*total_space = st.f_bsize * (uint64_t)st.f_blocks;
if (free_space)
*free_space = st.f_bsize * (uint64_t)st.f_bfree;
#else
@ -257,6 +259,8 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space)
/* by default assume yes */
if (has_persistent_inode)
*has_persistent_inode = 1;
if (total_space)
*total_space = 0;
if (free_space)
*free_space = 0;
#endif