diff --git a/cmdline/elem.c b/cmdline/elem.c index 6d6b4b70..4b0edeeb 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -649,6 +649,8 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev disk->device = dev; disk->tick = 0; + disk->total_blocks = 0; + disk->free_blocks = 0; disk->first_free_block = 0; disk->has_volatile_inodes = 0; disk->has_unreliable_physical = 0; @@ -727,14 +729,16 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) return 1; } -struct snapraid_map* map_alloc(const char* name, unsigned position, const char* uuid) +struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid) { struct snapraid_map* map; map = malloc_nofail(sizeof(struct snapraid_map)); pathcpy(map->name, sizeof(map->name), name); - pathcpy(map->uuid, sizeof(map->uuid), uuid); map->position = position; + map->total_blocks = total_blocks; + map->free_blocks = free_blocks; + pathcpy(map->uuid, sizeof(map->uuid), uuid); return map; } diff --git a/cmdline/elem.h b/cmdline/elem.h index 48751f03..1b3435fc 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -308,6 +308,9 @@ struct snapraid_disk { uint64_t tick; /**< Usage time of the disk. */ + block_off_t total_blocks; /**< Number of total blocks. */ + block_off_t free_blocks; /**< Number of free blocks at the last sync. */ + /** * First free searching block. * Note that it doesn't necessarely point at the first free block, @@ -364,8 +367,10 @@ struct snapraid_disk { */ struct snapraid_map { char name[PATH_MAX]; /**< Name of the disk. */ - unsigned position; /**< Position of the disk in the parity. */ char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */ + block_off_t total_blocks; /**< Number of total blocks. */ + block_off_t free_blocks; /**< Number of free blocks at last 'sync'. */ + unsigned position; /**< Position of the disk in the parity. */ /* nodes for data structures */ tommy_node node; @@ -376,7 +381,15 @@ struct snapraid_map { */ struct snapraid_parity { char path[PATH_MAX]; /**< Path of the parity file. */ + char uuid[UUID_MAX]; /**< UUID of the disk. Empty if unknown. */ uint64_t device; /**< Device identifier of the parity. */ + block_off_t total_blocks; /**< Number of total blocks. */ + block_off_t free_blocks; /**< Number of free blocks at the last sync. */ + + /** + * Cumulative time used for parity disks. + */ + uint64_t tick; }; /** @@ -869,7 +882,7 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax); * Allocates a disk mapping. * Uses uuid="" if not available. */ -struct snapraid_map* map_alloc(const char* name, unsigned position, const char* uuid); +struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid); /** * Deallocates a disk mapping. diff --git a/cmdline/mingw.c b/cmdline/mingw.c index 41d8529e..6f3c76ca 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -1431,13 +1431,14 @@ int filephy(const char* file, uint64_t size, uint64_t* physical) return 0; } -int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space) +int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space) { /* all FAT/exFAT/NTFS when managed from Windows have persistent inodes */ if (has_persistent_inode) *has_persistent_inode = 1; - if (free_space) { + if (free_space || total_space) { + ULARGE_INTEGER total_bytes; ULARGE_INTEGER total_free_bytes; DWORD attr; char dir[PATH_MAX]; @@ -1484,12 +1485,15 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space) /* get the free space of the directory */ /* note that it must be a directory */ - if (!GetDiskFreeSpaceExW(convert(dir), 0, 0, &total_free_bytes)) { + if (!GetDiskFreeSpaceExW(convert(dir), 0, &total_bytes, &total_free_bytes)) { windows_errno(GetLastError()); return -1; } - *free_space = total_free_bytes.QuadPart; + if (total_space) + *total_space = total_bytes.QuadPart; + if (free_space) + *free_space = total_free_bytes.QuadPart; } return 0; diff --git a/cmdline/portable.h b/cmdline/portable.h index cbbbcd3e..9566b64c 100644 --- a/cmdline/portable.h +++ b/cmdline/portable.h @@ -243,7 +243,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical); * Checks if the underline filesystem support persistent inodes. * Returns -1 on error, 0 on success. */ -int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space); +int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space); /** * Get the tick counter value. diff --git a/cmdline/scan.c b/cmdline/scan.c index 73b20f5f..745ee26c 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -1384,7 +1384,7 @@ void state_scan(struct snapraid_state* state, int is_diff) printf("Scanning disk %s...\n", disk->name); /* check if the disk supports persistent inodes */ - ret = fsinfo(disk->dir, &has_persistent_inode, 0); + ret = fsinfo(disk->dir, &has_persistent_inode, 0, 0); if (ret < 0) { /* LCOV_EXCL_START */ fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno)); diff --git a/cmdline/scrub.c b/cmdline/scrub.c index ed4c242f..8148bdc4 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -629,6 +629,9 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) printf("Scrubbing...\n"); + /* after opening parity files, refresh the free info */ + state_refresh(state); + error = 0; ret = state_scrub_process(state, parity_ptr, 0, blockmax, timelimit, lastlimit, now); diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 1ad1c222..5346fb3b 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -237,6 +237,7 @@ void config(char* conf, size_t conf_size, const char* argv0) #define OPT_TEST_FORCE_SCAN_WINFIND 279 #define OPT_TEST_IMPORT_CONTENT 280 #define OPT_TEST_FORCE_PROGRESS 281 +#define OPT_TEST_SKIP_DISK_ACCESS 282 #if HAVE_GETOPT_LONG struct option long_options[] = { @@ -341,6 +342,9 @@ struct option long_options[] = { /* Force immediate progress state update */ { "test-force-progress", 0, 0, OPT_TEST_FORCE_PROGRESS }, + /* Skip the disk access */ + { "test-skip-disk-access", 0, 0, OPT_TEST_SKIP_DISK_ACCESS }, + { 0, 0, 0, 0 } }; #endif @@ -604,6 +608,9 @@ int main(int argc, char* argv[]) case OPT_TEST_SKIP_PARITY_ACCESS : opt.skip_parity_access = 1; break; + case OPT_TEST_SKIP_DISK_ACCESS : + opt.skip_disk_access = 1; + break; case OPT_TEST_FORCE_MURMUR3 : opt.force_murmur3 = 1; break; @@ -828,6 +835,17 @@ int main(int argc, char* argv[]) } } + switch (operation) { + case OPERATION_LIST : + case OPERATION_DUP : + case OPERATION_STATUS : + case OPERATION_REWRITE : + case OPERATION_REHASH : + /* avoid to check and access data disks if not needed */ + opt.skip_disk_access = 1; + break; + } + switch (operation) { case OPERATION_DIFF : case OPERATION_LIST : @@ -835,6 +853,7 @@ int main(int argc, char* argv[]) case OPERATION_POOL : case OPERATION_STATUS : case OPERATION_REWRITE : + case OPERATION_REHASH : case OPERATION_NANO : /* avoid to check and access parity disks if not needed */ opt.skip_parity_access = 1; diff --git a/cmdline/state.c b/cmdline/state.c index 4ff1de0b..9262a488 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -85,8 +85,11 @@ void state_init(struct snapraid_state* state) state->file_mode = MODE_SEQUENTIAL; for (l = 0; l < LEV_MAX; ++l) { state->parity[l].path[0] = 0; + state->parity[l].uuid[0] = 0; state->parity[l].device = 0; - state->tick_parity[l] = 0; + state->parity[l].total_blocks = 0; + state->parity[l].free_blocks = 0; + state->parity[l].tick = 0; } state->tick_io = 0; state->tick_cpu = 0; @@ -173,8 +176,8 @@ static void state_config_check(struct snapraid_state* state, const char* path, t } } - /* check if all the data and parity disks are different */ - if (!state->opt.skip_device) { + /* check device of data disks */ + if (!state->opt.skip_device && !state->opt.skip_disk_access) { for (i = state->disklist; i != 0; i = i->next) { tommy_node* j; struct snapraid_disk* disk = i->data; @@ -212,17 +215,19 @@ static void state_config_check(struct snapraid_state* state, const char* path, t } } - for (l = 0; l < state->level; ++l) { - if (disk->device == state->parity[l].device) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].path); + if (!state->opt.skip_parity_access) { + for (l = 0; l < state->level; ++l) { + if (disk->device == state->parity[l].device) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Disk '%s' and %s '%s' are on the same device.\n", disk->dir, lev_name(l), state->parity[l].path); #ifdef _WIN32 - fprintf(stderr, "Both have the serial number '%" PRIx64 "'.\n", disk->device); - fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n"); - fprintf(stderr, "to change one of the disk serial.\n"); + fprintf(stderr, "Both have the serial number '%" PRIx64 "'.\n", disk->device); + fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'\n"); + fprintf(stderr, "to change one of the disk serial.\n"); #endif - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } } @@ -238,7 +243,10 @@ static void state_config_check(struct snapraid_state* state, const char* path, t /* LCOV_EXCL_STOP */ } } + } + /* check device of parity disks */ + if (!state->opt.skip_device && !state->opt.skip_parity_access) { #ifdef _WIN32 for (l = 0; l < state->level; ++l) { if (state->parity[l].device == 0) { @@ -251,16 +259,6 @@ static void state_config_check(struct snapraid_state* state, const char* path, t /* LCOV_EXCL_STOP */ } } - - if (state->pool[0] != 0 && state->pool_device == 0) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Disk '%s' has a zero serial number.\n", state->pool); - fprintf(stderr, "This is not necessarely wrong, but for using SnapRAID\n"); - fprintf(stderr, "it's better to change the serial number of the disk.\n"); - fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } #endif for (l = 0; l < state->level; ++l) { @@ -295,6 +293,21 @@ static void state_config_check(struct snapraid_state* state, const char* path, t } } + /* check device of pool disk */ +#ifdef _WIN32 + if (!state->opt.skip_device) { + if (state->pool[0] != 0 && state->pool_device == 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Disk '%s' has a zero serial number.\n", state->pool); + fprintf(stderr, "This is not necessarely wrong, but for using SnapRAID\n"); + fprintf(stderr, "it's better to change the serial number of the disk.\n"); + fprintf(stderr, "Try using the 'VolumeID' tool by 'Mark Russinovich'.\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } +#endif + /* count the content files */ if (!state->opt.skip_device) { unsigned content_count; @@ -529,7 +542,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co state->parity[level].device = st.st_dev; } else { /* if parity is skipped, uses a fake device */ - state->parity[level].device = -1LL - level; + state->parity[level].device = 0; } /* adjust the level */ @@ -662,8 +675,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co } else if (strcmp(tag, "disk") == 0) { char dir[PATH_MAX]; char device[PATH_MAX]; - struct stat st; struct snapraid_disk* disk; + uint64_t dev; ret = sgettok(f, buffer, sizeof(buffer)); if (ret < 0) { @@ -699,12 +712,6 @@ void state_config(struct snapraid_state* state, const char* path, const char* co /* get the device of the dir */ pathimport(device, sizeof(device), dir); - if (stat(device, &st) != 0) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } /* check if the disk name already exists */ for (i = state->disklist; i != 0; i = i->next) { @@ -719,7 +726,23 @@ void state_config(struct snapraid_state* state, const char* path, const char* co /* LCOV_EXCL_STOP */ } - disk = disk_alloc(buffer, dir, st.st_dev); + if (!state->opt.skip_disk_access) { + struct stat st; + + if (stat(device, &st) != 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + dev = st.st_dev; + } else { + /* if data disks are skipped, uses a fake device */ + dev = 0; + } + + disk = disk_alloc(buffer, dir, dev); tommy_list_insert_tail(&state->disklist, &disk->node, disk); } else if (strcmp(tag, "nohidden") == 0) { @@ -930,6 +953,7 @@ static void state_map(struct snapraid_state* state) tommy_node* i; unsigned uuid_mismatch; unsigned diskcount; + unsigned l; /* removes all the mapping without a disk */ /* this happens when a disk is removed from the configuration file */ @@ -967,8 +991,12 @@ static void state_map(struct snapraid_state* state) break; } } - if (j != 0) + if (j != 0) { + /* mapping is present, then copy the free blocks into to disk */ + disk->total_blocks = map->total_blocks; + disk->free_blocks = map->free_blocks; continue; + } /* mapping not found, search for an hole */ while (1) { @@ -989,7 +1017,7 @@ static void state_map(struct snapraid_state* state) } /* insert the new mapping */ - map = map_alloc(disk->name, hole, ""); + map = map_alloc(disk->name, hole, 0, 0, ""); tommy_list_insert_tail(&state->maplist, &map->node, map); } @@ -998,50 +1026,73 @@ static void state_map(struct snapraid_state* state) if (state->no_conf) return; - /* checks if mapping match the disk uuid */ uuid_mismatch = 0; - for (i = state->maplist; i != 0; i = i->next) { - struct snapraid_map* map = i->data; - struct snapraid_disk* disk; - char uuid[UUID_MAX]; - int ret; - disk = find_disk(state, map->name); - if (disk == 0) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } + /* checks if mapping match the disk uuid */ + if (!state->opt.skip_disk_access) { + for (i = state->maplist; i != 0; i = i->next) { + struct snapraid_map* map = i->data; + struct snapraid_disk* disk; + char uuid[UUID_MAX]; + int ret; - ret = devuuid(disk->device, uuid, sizeof(uuid)); - if (ret != 0) { - /* uuid not available, just ignore but marks the disk with unsupported UUID */ - disk->has_unsupported_uuid = 1; - continue; - } - - /* if the uuid is changed */ - if (strcmp(uuid, map->uuid) != 0) { - /* mark the disk as with an UUID change */ - disk->has_different_uuid = 1; - - /* if the previous uuid is available */ - if (map->uuid[0] != 0) { - /* count the number of uuid change */ - ++uuid_mismatch; - fprintf(stderr, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, uuid); - } else { - /* no message here, because having a disk without */ - /* UUID is the normal state of an empty disk */ - disk->had_empty_uuid = 1; + disk = find_disk(state, map->name); + if (disk == 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ } - /* update the uuid in the mapping, */ - pathcpy(map->uuid, sizeof(map->uuid), uuid); + ret = devuuid(disk->device, uuid, sizeof(uuid)); + if (ret != 0) { + /* uuid not available, just ignore but marks the disk with unsupported UUID */ + disk->has_unsupported_uuid = 1; + continue; + } - /* write the new state with the new uuid */ - state->need_write = 1; + /* if the uuid is changed */ + if (strcmp(uuid, map->uuid) != 0) { + /* mark the disk as with an UUID change */ + disk->has_different_uuid = 1; + + /* if the previous uuid is available */ + if (map->uuid[0] != 0) { + /* count the number of uuid change */ + ++uuid_mismatch; + fprintf(stderr, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, uuid); + } else { + /* no message here, because having a disk without */ + /* UUID is the normal state of an empty disk */ + disk->had_empty_uuid = 1; + } + + /* update the uuid in the mapping, */ + pathcpy(map->uuid, sizeof(map->uuid), uuid); + + /* write the new state with the new uuid */ + state->need_write = 1; + } + } + } + + /* checks the parity uuid */ + if (!state->opt.skip_parity_access) { + for (l = 0; l < state->level; ++l) { + char uuid[UUID_MAX]; + int ret; + + ret = devuuid(state->parity[l].device, uuid, sizeof(uuid)); + if (ret != 0) { + /* uuid not available, just ignore */ + continue; + } + + /* update the uuid in the mapping */ + /* at now we don't enforce uuid for parity */ + /* as the user is expected to have a different file name */ + /* for each parity file */ + pathcpy(state->parity[l].uuid, sizeof(state->parity[l].uuid), uuid); } } @@ -1085,6 +1136,67 @@ static void state_map(struct snapraid_state* state) } } +void state_refresh(struct snapraid_state* state) +{ + tommy_node* i; + unsigned l; + + /* for all disks */ + for (i = state->maplist; i != 0; i = i->next) { + struct snapraid_map* map = i->data; + struct snapraid_disk* disk; + uint64_t total_space; + uint64_t free_space; + int ret; + + disk = find_disk(state, map->name); + if (disk == 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Internal inconsistency for mapping '%s'\n", map->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = fsinfo(disk->dir, 0, &total_space, &free_space); + if (ret != 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* set the new free blocks */ + map->total_blocks = total_space / state->block_size; + map->free_blocks = free_space / state->block_size; + + /* also update the disk info */ + disk->total_blocks = map->total_blocks; + disk->free_blocks = map->free_blocks; + } + + /* for all parities */ + for (l = 0; l < state->level; ++l) { + uint64_t total_space; + uint64_t free_space; + int ret; + + ret = fsinfo(state->parity[l].path, 0, &total_space, &free_space); + if (ret != 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error accessing file '%s' to get filesystem info. %s.\n", state->parity[l].path, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* set the new free blocks */ + state->parity[l].total_blocks = total_space / state->block_size; + state->parity[l].free_blocks = free_space / state->block_size; + } + + /* note what we don't set need_write = 1, because we don't want */ + /* to update the content file only for the free space info. */ +} + /** * Checks the content. */ @@ -2020,10 +2132,12 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - } else if (strcmp(tag, "map") == 0) { + } else if (strcmp(tag, "map") == 0 || strcmp(tag, "mapping") == 0) { struct snapraid_map* map; char uuid[UUID_MAX]; uint32_t v_pos; + uint32_t v_total_blocks; + uint32_t v_free_blocks; ret = sgettok(f, buffer, sizeof(buffer)); if (ret < 0) { @@ -2049,6 +2163,46 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE /* LCOV_EXCL_STOP */ } + /* from SnapRAID 7.0 the 'mapping' command includes the free space */ + if (strcmp(tag, "mapping") == 0) { + c = sgetc(f); + if (c != ' ') { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetu32(f, &v_total_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + c = sgetc(f); + if (c != ' ') { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetu32(f, &v_free_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } else { + v_total_blocks = 0; + v_free_blocks = 0; + } + + /* check if the uuid is present */ + /* uuid must be the latest one as it could be also empty */ c = sgetc(f); if (c != ' ') { sungetc(c, f); @@ -2064,9 +2218,80 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } } - map = map_alloc(buffer, v_pos, uuid); + map = map_alloc(buffer, v_pos, v_total_blocks, v_free_blocks, uuid); tommy_list_insert_tail(&state->maplist, &map->node, map); + } else if (strcmp(tag, "parity") == 0) { + /* from SnapRAID 7.0 the 'parity' command includes the free space */ + char uuid[UUID_MAX]; + uint32_t v_level; + uint32_t v_total_blocks; + uint32_t v_free_blocks; + + ret = sgetu32(f, &v_level); + if (ret < 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + c = sgetc(f); + if (c != ' ') { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetu32(f, &v_total_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + c = sgetc(f); + if (c != ' ') { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetu32(f, &v_free_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* check if the uuid is present */ + /* uuid must be the latest one as it could be also empty */ + c = sgetc(f); + if (c != ' ') { + sungetc(c, f); + uuid[0] = 0; + } else { + /* read the uuid */ + ret = sgettok(f, uuid, sizeof(uuid)); + if (ret < 0) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Invalid 'map' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + + /* if we use this parity entry */ + if (v_level < state->level) { + /* set the parity info */ + pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid); + state->parity[v_level].total_blocks = v_total_blocks; + state->parity[v_level].free_blocks = v_free_blocks; + } } else if (strcmp(tag, "sign") == 0) { uint32_t sign; @@ -2174,6 +2399,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) block_off_t b; block_off_t blockmax; int info_has_rehash; + unsigned l; count_file = 0; count_hardlink = 0; @@ -2277,16 +2503,21 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) /* save the mapping only for not empty disks */ if (!disk_is_empty(disk, blockmax)) { - sputsl("map ", f); + sputsl("mapping ", f); sputs(map->name, f); sputc(' ', f); sputu32(map->position, f); + sputc(' ', f); + sputu32(map->total_blocks, f); + sputc(' ', f); + sputu32(map->free_blocks, f); /* if there is an uuid, print it */ if (map->uuid[0]) { sputc(' ', f); sputs(map->uuid, f); } + sputeol(f); if (serror(f)) { /* LCOV_EXCL_START */ @@ -2303,6 +2534,30 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) } } + /* for each parity */ + for (l = 0; l < state->level; ++l) { + sputsl("parity ", f); + sputu32(l, f); + sputc(' ', f); + sputu32(state->parity[l].total_blocks, f); + sputc(' ', f); + sputu32(state->parity[l].free_blocks, f); + + /* if there is an uuid, print it */ + if (state->parity[l].uuid[0]) { + sputc(' ', f); + sputs(state->parity[l].uuid, f); + } + + sputeol(f); + if (serror(f)) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + /* for each disk */ for (i = state->disklist; i != 0; i = i->next) { tommy_node* j; @@ -2625,7 +2880,16 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0) { + + /* + * File format versions: + * - SNAPCNT1/SnapRAID 4.0 First version. + * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support. + * The previous 'm' entry is now deprecated, but supported for importing. + * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'. + */ + if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0 + && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0) { /* LCOV_EXCL_START */ if (memcmp(buffer, "SNAPCNT", 7) != 0) { decoding_error(path, f); @@ -3335,10 +3599,12 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - } else if (c == 'm') { + } else if (c == 'm' || c == 'M') { struct snapraid_map* map; char uuid[UUID_MAX]; uint32_t v_pos; + uint32_t v_total_blocks; + uint32_t v_free_blocks; struct snapraid_disk* disk; ret = sgetbs(f, buffer, sizeof(buffer)); @@ -3357,6 +3623,28 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } + /* from SnapRAID 7.0 the 'M' command includes the free space */ + if (c == 'M') { + ret = sgetb32(f, &v_total_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetb32(f, &v_free_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } else { + v_total_blocks = 0; + v_free_blocks = 0; + } + /* read the uuid */ ret = sgetbs(f, uuid, sizeof(uuid)); if (ret < 0) { @@ -3366,7 +3654,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } - map = map_alloc(buffer, v_pos, uuid); + map = map_alloc(buffer, v_pos, v_total_blocks, v_free_blocks, uuid); tommy_list_insert_tail(&state->maplist, &map->node, map); @@ -3385,6 +3673,52 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST tommy_array_grow(&disk_mapping, mapping_max + 1); tommy_array_set(&disk_mapping, mapping_max, disk); ++mapping_max; + } else if (c == 'P') { + /* from SnapRAID 7.0 the 'P' command includes the free space */ + char uuid[UUID_MAX]; + uint32_t v_level; + uint32_t v_total_blocks; + uint32_t v_free_blocks; + + ret = sgetb32(f, &v_level); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetb32(f, &v_total_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetb32(f, &v_free_blocks); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + ret = sgetbs(f, uuid, sizeof(uuid)); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* if we use this parity entry */ + if (v_level < state->level) { + /* set the parity info */ + pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid); + state->parity[v_level].total_blocks = v_total_blocks; + state->parity[v_level].free_blocks = v_free_blocks; + } } else if (c == 'N') { uint32_t crc_stored; uint32_t crc_computed; @@ -3475,6 +3809,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) time_t info_oldest; int info_has_rehash; int mapping_idx; + unsigned l; count_file = 0; count_hardlink = 0; @@ -3513,7 +3848,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) } /* write header */ - swrite("SNAPCNT1\n\3\0\0", 12, f); + swrite("SNAPCNT2\n\3\0\0", 12, f); /* write block size and block max */ sputc('z', f); @@ -3588,9 +3923,11 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* save the mapping only for not empty disks */ if (!disk_is_empty(disk, blockmax)) { - sputc('m', f); + sputc('M', f); sputbs(map->name, f); sputb32(map->position, f); + sputb32(map->total_blocks, f); + sputb32(map->free_blocks, f); sputbs(map->uuid, f); if (serror(f)) { /* LCOV_EXCL_START */ @@ -3608,6 +3945,21 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) } } + /* for each parity */ + for (l = 0; l < state->level; ++l) { + sputc('P', f); + sputb32(l, f); + sputb32(state->parity[l].total_blocks, f); + sputb32(state->parity[l].free_blocks, f); + sputbs(state->parity[l].uuid, f); + if (serror(f)) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + /* for each disk */ for (i = state->disklist; i != 0; i = i->next) { tommy_node* j; @@ -4443,9 +4795,9 @@ void state_usage_print(struct snapraid_state* state) tick_max = disk->tick; } for (l = 0; l < state->level; ++l) { - tick_total += state->tick_parity[l]; - if (state->tick_parity[l] > tick_max) - tick_max = state->tick_parity[l]; + tick_total += state->parity[l].tick; + if (state->parity[l].tick > tick_max) + tick_max = state->parity[l].tick; } if (!tick_total) @@ -4464,11 +4816,11 @@ void state_usage_print(struct snapraid_state* state) printf("Time for parity:"); for (l = 0; l < state->level; ++l) { - if (state->tick_parity[l] == tick_max) + if (state->parity[l].tick == tick_max) printf(" %s:", lev_config_name(l)); else printf(" "); - printf("%" PRIu64 "%%", state->tick_parity[l] * 100U / tick_total); + printf("%" PRIu64 "%%", state->parity[l].tick * 100U / tick_total); } printf("\n"); } diff --git a/cmdline/state.h b/cmdline/state.h index 0e640f02..8e19512f 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -82,6 +82,7 @@ struct snapraid_option { int skip_self; /**< Skip the self-test. */ int skip_content_check; /**< Relax some content file checks. */ int skip_parity_access; /**< Skip the parity access for commands that don't need it. */ + int skip_disk_access; /**< Skip the data disk access for commands that don't need it. */ int kill_after_sync; /**< Kill the process after sync without saving the final state. */ int force_murmur3; /**< Force Murmur3 choice. */ int force_spooky2; /**< Force Spooky2 choice. */ @@ -140,11 +141,6 @@ struct snapraid_state { */ uint64_t tick_io; - /** - * Cumulative time used for parity disks. - */ - uint64_t tick_parity[LEV_MAX]; - /** * Last time used for time measure. */ @@ -263,6 +259,13 @@ void state_list(struct snapraid_state* state); */ void state_pool(struct snapraid_state* state); +/** + * Refresh the free space info. + * + * Note that it requires disks access. + */ +void state_refresh(struct snapraid_state* state); + /** * Filter files, symlinks and dirs. * Apply an additional filter to the list currently loaded. @@ -344,7 +347,7 @@ static inline void state_usage_parity(struct snapraid_state* state, unsigned lev assert(level < LEV_MAX); /* increment the time spent in the parity disk */ - state->tick_parity[level] += delta; + state->parity[level].tick += delta; state->tick_io += delta; state->tick_last = now; diff --git a/cmdline/status.c b/cmdline/status.c index 1e19fdd4..1728e0da 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -52,6 +52,7 @@ int state_status(struct snapraid_state* state) block_off_t bad; block_off_t rehash; block_off_t count; + unsigned l; unsigned dayoldest, daymedian, daynewest; unsigned bar[GRAPH_COLUMN]; unsigned barpos; @@ -68,9 +69,7 @@ int state_status(struct snapraid_state* state) uint64_t base = 1024 * 1024 * (uint64_t)1024; block_off_t parity_block_free; unsigned unsynced_blocks; - uint64_t free_space; uint64_t all_wasted; - int ret; /* get the present time */ now = time(0); @@ -80,18 +79,15 @@ int state_status(struct snapraid_state* state) fprintf(stdlog, "summary:block_size:%u\n", state->block_size); fprintf(stdlog, "summary:parity_block_count:%u\n", blockmax); - /* get the free space on the first parity disk */ - ret = fsinfo(state->parity[0].path, 0, &free_space); - if (ret < 0) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Error accessing file '%s' to get filesystem info. %s.\n", state->parity[0].path, strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + /* get the minimum parity free space */ + parity_block_free = state->parity[0].free_blocks; + for (l = 0; l < state->level; ++l) { + fprintf(stdlog, "summary:parity_block_total:%s:%u\n", lev_config_name(l), state->parity[l].total_blocks); + fprintf(stdlog, "summary:parity_block_free:%s:%u\n", lev_config_name(l), state->parity[l].free_blocks); + if (state->parity[l].free_blocks < parity_block_free) + parity_block_free = state->parity[l].free_blocks; } - - parity_block_free = free_space / state->block_size; - - fprintf(stdlog, "summary:parity_block_free:%s:%u\n", lev_config_name(0), parity_block_free); + fprintf(stdlog, "summary:parity_block_free_min:%u\n", parity_block_free); printf("\n"); printf(" Files Fragmented Excess Wasted Used Free Use Name\n"); @@ -115,7 +111,6 @@ int state_status(struct snapraid_state* state) block_off_t disk_block_count = 0; uint64_t disk_file_size = 0; block_off_t disk_block_latest_used = 0; - block_off_t disk_block_free; block_off_t disk_block_max_by_space; block_off_t disk_block_max_by_parity; block_off_t disk_block_max; @@ -167,18 +162,8 @@ int state_status(struct snapraid_state* state) disk_file_size += file->size; } - /* get the free space on the disk */ - ret = fsinfo(disk->dir, 0, &free_space); - if (ret < 0) { - /* LCOV_EXCL_START */ - fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - disk_block_free = free_space / state->block_size; - - disk_block_max_by_space = disk_block_count + disk_block_free; + /* get the free block info */ + disk_block_max_by_space = disk_block_count + disk->free_blocks; disk_block_max_by_parity = blockmax + parity_block_free; /* the maximum usable space in a disk is limited by the smaller */ @@ -213,7 +198,8 @@ int state_status(struct snapraid_state* state) fprintf(stdlog, "summary:disk_excess_fragment_count:%s:%u\n", disk->name, disk_extra_fragment); fprintf(stdlog, "summary:disk_file_size:%s:%" PRIu64 "\n", disk->name, disk_file_size); fprintf(stdlog, "summary:disk_block_allocated:%s:%u\n", disk->name, disk_block_latest_used + 1); - fprintf(stdlog, "summary:disk_block_free:%s:%u\n", disk->name, disk_block_free); + fprintf(stdlog, "summary:disk_block_total:%s:%u\n", disk->name, disk->total_blocks); + fprintf(stdlog, "summary:disk_block_free:%s:%u\n", disk->name, disk->free_blocks); fprintf(stdlog, "summary:disk_block_max_by_space:%s:%u\n", disk->name, disk_block_max_by_space); fprintf(stdlog, "summary:disk_block_max_by_parity:%s:%u\n", disk->name, disk_block_max_by_parity); fprintf(stdlog, "summary:disk_block_max:%s:%u\n", disk->name, disk_block_max); diff --git a/cmdline/sync.c b/cmdline/sync.c index 9b8449f8..54c901c2 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1090,6 +1090,9 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t } } + /* after opening and resizing parity files, refresh the free info */ + state_refresh(state); + unrecoverable_error = 0; if (state->opt.prehash) { diff --git a/cmdline/unix.c b/cmdline/unix.c index 1034f396..3a71c1e3 100644 --- a/cmdline/unix.c +++ b/cmdline/unix.c @@ -205,7 +205,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical) return 0; } -int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space) +int fsinfo(const char* path, int* has_persistent_inode, uint64_t* total_space, uint64_t* free_space) { #if HAVE_STATFS && HAVE_STRUCT_STATFS_F_TYPE struct statfs st; @@ -249,6 +249,8 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space) break; } + if (total_space) + *total_space = st.f_bsize * (uint64_t)st.f_blocks; if (free_space) *free_space = st.f_bsize * (uint64_t)st.f_bfree; #else @@ -257,6 +259,8 @@ int fsinfo(const char* path, int* has_persistent_inode, uint64_t* free_space) /* by default assume yes */ if (has_persistent_inode) *has_persistent_inode = 1; + if (total_space) + *total_space = 0; if (free_space) *free_space = 0; #endif