diff --git a/TODO b/TODO index 9b2214c9..23df6abd 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,8 @@ SnapRAID TODO This is the list of TODO items for SnapRAID. +* Check blockmax in hole ? RISKY ?? + - v4.x * If a content file is missing, sync should rewrite it, even if no change was done at the content. diff --git a/elem.c b/elem.c index 7d25364e..5fed17e3 100644 --- a/elem.c +++ b/elem.c @@ -523,6 +523,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev disk->device = dev; disk->first_free_block = 0; + disk->mapping = -1; tommy_list_init(&disk->filelist); tommy_list_init(&disk->deletedlist); tommy_hashdyn_init(&disk->inodeset); @@ -550,6 +551,46 @@ void disk_free(struct snapraid_disk* disk) free(disk); } +int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) +{ + block_off_t i; + + /* if there is an element, it's not empty */ + /* even if links and dirs have no block allocation */ + if (!tommy_list_empty(&disk->filelist)) + return 0; + if (!tommy_list_empty(&disk->linklist)) + return 0; + if (!tommy_list_empty(&disk->dirlist)) + return 0; + + /* limit the search to the size of the disk */ + if (blockmax > tommy_array_size(&disk->blockarr)) + blockmax = tommy_array_size(&disk->blockarr); + + /* checks all the blocks to search for deleted ones */ + /* this search is slow, but it's done only if no file is present */ + for(i=0;iblockarr, i); + unsigned block_state = block_state_get(block); + + switch (block_state) { + case BLOCK_STATE_EMPTY : + /* empty block are expected for an empty disk */ + break; + case BLOCK_STATE_DELETED : + /* if there is a deleted block, the disk is not empty */ + return 0; + default: + fprintf(stderr, "Internal incosistency for used block in disk '%s' without files\n", disk->name); + exit(EXIT_FAILURE); + } + } + + /* finally, it's empty */ + return 1; +} + struct snapraid_map* map_alloc(const char* name, unsigned position, const char* uuid) { struct snapraid_map* map; diff --git a/elem.h b/elem.h index a9adf1e0..dcb80e64 100644 --- a/elem.h +++ b/elem.h @@ -70,7 +70,7 @@ struct snapraid_filter { * The block has both the hash and the parity computed. * This is the normal state of a saved block. * Note that if exists at least one BLOCK_STATE_BLK in a disk, all the other - * blocks at the same address in other disks can be only BLOCK_EMPTY or BLOCK_STATE_BLK. + * blocks at the same address in other disks can be only BLOCK_STATE_EMPTY or BLOCK_STATE_BLK. * * The block hash field IS set. * The parity for this disk is updated. @@ -264,6 +264,7 @@ struct snapraid_disk { char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */ uint64_t device; /**< Device identifier. */ block_off_t first_free_block; /**< First free searching block. */ + int mapping; /**< Index in the mapping vector. Used only as buffer when writing the content file. */ /**< * Block array of the disk. @@ -718,6 +719,14 @@ static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, return BLOCK_EMPTY; } +/** + * Check if a disk is totally empty and can be discarded from the content file. + * A disk is empty if it doesn't contain any file, symlink, hardlink or dir + * and without any DELETED block. + * The blockmax is used to limit the search of DELETED block up to blockmax. + */ +int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax); + /** * Allocates a disk mapping. * Uses uuid="" if not available. diff --git a/state.c b/state.c index 0941b03c..98fe6557 100644 --- a/state.c +++ b/state.c @@ -1723,28 +1723,49 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) /* for each map */ for(i=state->maplist;i!=0;i=i->next) { struct snapraid_map* map = i->data; - sputsl("map ", f); - sputs(map->name, f); - sputc(' ', f); - sputu32(map->position, f); + struct snapraid_disk* disk; - /* if there is an uuid, print it */ - if (map->uuid[0]) { - sputc(' ', f); - sputs(map->uuid, f); - } - sputeol(f); - if (serror(f)) { - fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + /* find the disk for this mapping */ + disk = find_disk(state, map->name); + if (!disk) { + fprintf(stderr, "Internal incosistency for unmapped disk '%s'\n", map->name); exit(EXIT_FAILURE); } + + /* save the mapping only for not empty disks */ + if (!disk_is_empty(disk, blockmax)) { + sputsl("map ", f); + sputs(map->name, f); + sputc(' ', f); + sputu32(map->position, f); + + /* if there is an uuid, print it */ + if (map->uuid[0]) { + sputc(' ', f); + sputs(map->uuid, f); + } + sputeol(f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + /* mark the disk as with mapping */ + disk->mapping = 0; + } else { + /* mark the disk as without mapping */ + disk->mapping = -1; + } } /* for each disk */ for(i=state->disklist;i!=0;i=i->next) { tommy_node* j; struct snapraid_disk* disk = i->data; - int first_deleted; + + /* if the disk is not mapped, skip it */ + if (disk->mapping < 0) + continue; /* for each file */ for(j=disk->filelist;j!=0;j=j->next) { @@ -1864,24 +1885,18 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) ++count_dir; } - /* for each deleted block in the disk */ - first_deleted = 1; + /* deleted blocks of the disk */ + sputsl("hole ", f); + sputs(disk->name, f); + sputeol(f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } for(b=0;bname, f); - sputeol(f); - if (serror(f)) { - fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - } - } - sputsl("off ", f); sputu32(b, f); sputc(' ', f); @@ -2010,6 +2025,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST int crc_checked; char buffer[PATH_MAX]; int ret; + tommy_array disk_mapping; + uint32_t mapping_max; blockmax = 0; count_file = 0; @@ -2018,6 +2035,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST count_symlink = 0; count_dir = 0; crc_checked = 0; + mapping_max = 0; + tommy_array_init(&disk_mapping); ret = sread(f, buffer, 12); if (ret < 0 || memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0) { @@ -2046,12 +2065,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST uint32_t v_idx; struct snapraid_file* file; struct snapraid_disk* disk; + uint32_t mapping; - ret = sgetbs(f, buffer, sizeof(buffer)); - if (ret < 0) { + ret = sgetb32(f, &mapping); + if (ret < 0 || mapping >= mapping_max) { decoding_error(path, f); exit(EXIT_FAILURE); } + disk = tommy_array_get(&disk_mapping, mapping); ret = sgetb64(f, &v_size); if (ret < 0) { @@ -2104,14 +2125,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST exit(EXIT_FAILURE); } - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - decoding_error(path, f); - fprintf(stderr, "Disk named '%s' not found!\n", buffer); - exit(EXIT_FAILURE); - } - /* allocate the file */ file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0); @@ -2306,20 +2319,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* hole */ uint32_t v_pos; struct snapraid_disk* disk; + uint32_t mapping; - ret = sgetbs(f, buffer, sizeof(buffer)); - if (ret < 0) { + ret = sgetb32(f, &mapping); + if (ret < 0 || mapping >= mapping_max) { decoding_error(path, f); exit(EXIT_FAILURE); } - - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - decoding_error(path, f); - fprintf(stderr, "Disk named '%s' not found!\n", buffer); - exit(EXIT_FAILURE); - } + disk = tommy_array_get(&disk_mapping, mapping); /* grow the array */ tommy_array_grow(&disk->blockarr, blockmax); @@ -2396,12 +2403,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST char linkto[PATH_MAX]; struct snapraid_link* link; struct snapraid_disk* disk; + uint32_t mapping; - ret = sgetbs(f, buffer, sizeof(buffer)); - if (ret < 0) { + ret = sgetb32(f, &mapping); + if (ret < 0 || mapping >= mapping_max) { decoding_error(path, f); exit(EXIT_FAILURE); } + disk = tommy_array_get(&disk_mapping, mapping); ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { @@ -2420,14 +2429,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST exit(EXIT_FAILURE); } - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - decoding_error(path, f); - fprintf(stderr, "Disk named '%s' not found!\n", buffer); - exit(EXIT_FAILURE); - } - /* allocate the link as symbolic link */ link = link_alloc(sub, linkto, FILE_IS_SYMLINK); @@ -2443,12 +2444,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST char linkto[PATH_MAX]; struct snapraid_link* link; struct snapraid_disk* disk; + uint32_t mapping; - ret = sgetbs(f, buffer, sizeof(buffer)); - if (ret < 0) { + ret = sgetb32(f, &mapping); + if (ret < 0 || mapping >= mapping_max) { decoding_error(path, f); exit(EXIT_FAILURE); } + disk = tommy_array_get(&disk_mapping, mapping); ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { @@ -2467,14 +2470,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST exit(EXIT_FAILURE); } - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - decoding_error(path, f); - fprintf(stderr, "Disk named '%s' not found!\n", buffer); - exit(EXIT_FAILURE); - } - /* allocate the link as hard link */ link = link_alloc(sub, linkto, FILE_IS_HARDLINK); @@ -2489,12 +2484,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST char sub[PATH_MAX]; struct snapraid_dir* dir; struct snapraid_disk* disk; + uint32_t mapping; - ret = sgetbs(f, buffer, sizeof(buffer)); - if (ret < 0) { + ret = sgetb32(f, &mapping); + if (ret < 0 || mapping >= mapping_max) { decoding_error(path, f); exit(EXIT_FAILURE); } + disk = tommy_array_get(&disk_mapping, mapping); ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { @@ -2507,14 +2504,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST exit(EXIT_FAILURE); } - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - decoding_error(path, f); - fprintf(stderr, "Disk named '%s' not found!\n", buffer); - exit(EXIT_FAILURE); - } - /* allocate the dir */ dir = dir_alloc(sub); @@ -2595,6 +2584,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST struct snapraid_map* map; char uuid[UUID_MAX]; uint32_t v_pos; + struct snapraid_disk* disk; ret = sgetbs(f, buffer, sizeof(buffer)); if (ret < 0) { @@ -2618,6 +2608,19 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST map = map_alloc(buffer, v_pos, uuid); tommy_list_insert_tail(&state->maplist, &map->node, map); + + /* find the disk */ + disk = find_disk(state, buffer); + if (!disk) { + decoding_error(path, f); + fprintf(stderr, "Disk named '%s' not present in the configuration file!\n", buffer); + exit(EXIT_FAILURE); + } + + /* insert in the mapping vector */ + tommy_array_grow(&disk_mapping, mapping_max + 1); + tommy_array_set(&disk_mapping, mapping_max, disk); + ++mapping_max; } else if (c == 'N') { uint32_t crc_stored; uint32_t crc_computed; @@ -2648,6 +2651,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } } + tommy_array_done(&disk_mapping); + if (serror(f)) { fprintf(stderr, "Error reading the content file '%s' at offset %"PRIi64"\n", path, stell(f)); exit(EXIT_FAILURE); @@ -2688,6 +2693,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) block_off_t begin; time_t info_oldest; int info_has_rehash; + int mapping_idx; count_file = 0; count_block = 0; @@ -2773,16 +2779,36 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) } /* for each map */ + mapping_idx = 0; for(i=state->maplist;i!=0;i=i->next) { struct snapraid_map* map = i->data; - sputc('m', f); - sputbs(map->name, f); - sputb32(map->position, f); - sputbs(map->uuid, f); - if (serror(f)) { - fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + struct snapraid_disk* disk; + + /* find the disk for this mapping */ + disk = find_disk(state, map->name); + if (!disk) { + fprintf(stderr, "Internal incosistency for unmapped disk '%s'\n", map->name); exit(EXIT_FAILURE); } + + /* save the mapping only for not empty disks */ + if (!disk_is_empty(disk, blockmax)) { + sputc('m', f); + sputbs(map->name, f); + sputb32(map->position, f); + sputbs(map->uuid, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + + /* assign the mapping index used to identify disks */ + disk->mapping = mapping_idx; + ++mapping_idx; + } else { + /* mark the disk as without mapping */ + disk->mapping = -1; + } } /* for each disk */ @@ -2790,6 +2816,10 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) tommy_node* j; struct snapraid_disk* disk = i->data; + /* if the disk is not mapped, skip it */ + if (disk->mapping < 0) + continue; + /* for each file */ for(j=disk->filelist;j!=0;j=j->next) { struct snapraid_file* file = j->data; @@ -2805,7 +2835,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) inode = file->inode; sputc('f', f); - sputbs(disk->name, f); + sputb32(disk->mapping, f); sputb64(size, f); sputb64(mtime_sec, f); /* encode STAT_NSEC_INVALID as 0 */ @@ -2893,7 +2923,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) break; } - sputbs(disk->name, f); + sputb32(disk->mapping, f); sputbs(link->sub, f); sputbs(link->linkto, f); if (serror(f)) { @@ -2907,7 +2937,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) struct snapraid_dir* dir = j->data; sputc('r', f); - sputbs(disk->name, f); + sputb32(disk->mapping, f); sputbs(dir->sub, f); if (serror(f)) { fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); @@ -2917,7 +2947,13 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) ++count_dir; } - /* write the deleted info for each block */ + /* deleted blocks of the disk */ + sputc('h', f); + sputb32(disk->mapping, f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } begin = 0; while (begin < blockmax) { int is_deleted; @@ -2933,24 +2969,6 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) ++end; } - /* if it's the first iteration */ - if (begin == 0) { - /* if there is no deleted block in the disk, write nothing */ - /* this is required to avoid to have references to a completely empty disk */ - if (end == blockmax && !is_deleted) { - /* exit */ - break; - } - - /* write the hole header only if required */ - sputc('h', f); - sputbs(disk->name, f); - if (serror(f)) { - fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - } - } - sputb32(end - begin, f); if (is_deleted) {