mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
Writes indexes instead of disk names in the binary format.
Better detection of empty disks to remove the mapping record.
This commit is contained in:
Andrea Mazzoleni
parent
fb5cc1c3b4
commit
d141c7bc13
4 changed files with 176 additions and 106 deletions
2
TODO
2
TODO
|
|
@ -3,6 +3,8 @@ SnapRAID TODO
|
|||
|
||||
This is the list of TODO items for SnapRAID.
|
||||
|
||||
* Check blockmax in hole ? RISKY ??
|
||||
|
||||
- v4.x
|
||||
|
||||
* If a content file is missing, sync should rewrite it, even if no change was done at the content.
|
||||
|
|
|
|||
41
elem.c
41
elem.c
|
|
@ -523,6 +523,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
|
|||
|
||||
disk->device = dev;
|
||||
disk->first_free_block = 0;
|
||||
disk->mapping = -1;
|
||||
tommy_list_init(&disk->filelist);
|
||||
tommy_list_init(&disk->deletedlist);
|
||||
tommy_hashdyn_init(&disk->inodeset);
|
||||
|
|
@ -550,6 +551,46 @@ void disk_free(struct snapraid_disk* disk)
|
|||
free(disk);
|
||||
}
|
||||
|
||||
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax)
|
||||
{
|
||||
block_off_t i;
|
||||
|
||||
/* if there is an element, it's not empty */
|
||||
/* even if links and dirs have no block allocation */
|
||||
if (!tommy_list_empty(&disk->filelist))
|
||||
return 0;
|
||||
if (!tommy_list_empty(&disk->linklist))
|
||||
return 0;
|
||||
if (!tommy_list_empty(&disk->dirlist))
|
||||
return 0;
|
||||
|
||||
/* limit the search to the size of the disk */
|
||||
if (blockmax > tommy_array_size(&disk->blockarr))
|
||||
blockmax = tommy_array_size(&disk->blockarr);
|
||||
|
||||
/* checks all the blocks to search for deleted ones */
|
||||
/* this search is slow, but it's done only if no file is present */
|
||||
for(i=0;i<blockmax;++i) {
|
||||
struct snapraid_block* block = tommy_array_get(&disk->blockarr, i);
|
||||
unsigned block_state = block_state_get(block);
|
||||
|
||||
switch (block_state) {
|
||||
case BLOCK_STATE_EMPTY :
|
||||
/* empty block are expected for an empty disk */
|
||||
break;
|
||||
case BLOCK_STATE_DELETED :
|
||||
/* if there is a deleted block, the disk is not empty */
|
||||
return 0;
|
||||
default:
|
||||
fprintf(stderr, "Internal incosistency for used block in disk '%s' without files\n", disk->name);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
/* finally, it's empty */
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct snapraid_map* map_alloc(const char* name, unsigned position, const char* uuid)
|
||||
{
|
||||
struct snapraid_map* map;
|
||||
|
|
|
|||
11
elem.h
11
elem.h
|
|
@ -70,7 +70,7 @@ struct snapraid_filter {
|
|||
* The block has both the hash and the parity computed.
|
||||
* This is the normal state of a saved block.
|
||||
* Note that if exists at least one BLOCK_STATE_BLK in a disk, all the other
|
||||
* blocks at the same address in other disks can be only BLOCK_EMPTY or BLOCK_STATE_BLK.
|
||||
* blocks at the same address in other disks can be only BLOCK_STATE_EMPTY or BLOCK_STATE_BLK.
|
||||
*
|
||||
* The block hash field IS set.
|
||||
* The parity for this disk is updated.
|
||||
|
|
@ -264,6 +264,7 @@ struct snapraid_disk {
|
|||
char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
|
||||
uint64_t device; /**< Device identifier. */
|
||||
block_off_t first_free_block; /**< First free searching block. */
|
||||
int mapping; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
|
||||
|
||||
/**<
|
||||
* Block array of the disk.
|
||||
|
|
@ -718,6 +719,14 @@ static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk,
|
|||
return BLOCK_EMPTY;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a disk is totally empty and can be discarded from the content file.
|
||||
* A disk is empty if it doesn't contain any file, symlink, hardlink or dir
|
||||
* and without any DELETED block.
|
||||
* The blockmax is used to limit the search of DELETED block up to blockmax.
|
||||
*/
|
||||
int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax);
|
||||
|
||||
/**
|
||||
* Allocates a disk mapping.
|
||||
* Uses uuid="" if not available.
|
||||
|
|
|
|||
228
state.c
228
state.c
|
|
@ -1723,28 +1723,49 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
|
|||
/* for each map */
|
||||
for(i=state->maplist;i!=0;i=i->next) {
|
||||
struct snapraid_map* map = i->data;
|
||||
sputsl("map ", f);
|
||||
sputs(map->name, f);
|
||||
sputc(' ', f);
|
||||
sputu32(map->position, f);
|
||||
struct snapraid_disk* disk;
|
||||
|
||||
/* if there is an uuid, print it */
|
||||
if (map->uuid[0]) {
|
||||
sputc(' ', f);
|
||||
sputs(map->uuid, f);
|
||||
}
|
||||
sputeol(f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
/* find the disk for this mapping */
|
||||
disk = find_disk(state, map->name);
|
||||
if (!disk) {
|
||||
fprintf(stderr, "Internal incosistency for unmapped disk '%s'\n", map->name);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* save the mapping only for not empty disks */
|
||||
if (!disk_is_empty(disk, blockmax)) {
|
||||
sputsl("map ", f);
|
||||
sputs(map->name, f);
|
||||
sputc(' ', f);
|
||||
sputu32(map->position, f);
|
||||
|
||||
/* if there is an uuid, print it */
|
||||
if (map->uuid[0]) {
|
||||
sputc(' ', f);
|
||||
sputs(map->uuid, f);
|
||||
}
|
||||
sputeol(f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* mark the disk as with mapping */
|
||||
disk->mapping = 0;
|
||||
} else {
|
||||
/* mark the disk as without mapping */
|
||||
disk->mapping = -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* for each disk */
|
||||
for(i=state->disklist;i!=0;i=i->next) {
|
||||
tommy_node* j;
|
||||
struct snapraid_disk* disk = i->data;
|
||||
int first_deleted;
|
||||
|
||||
/* if the disk is not mapped, skip it */
|
||||
if (disk->mapping < 0)
|
||||
continue;
|
||||
|
||||
/* for each file */
|
||||
for(j=disk->filelist;j!=0;j=j->next) {
|
||||
|
|
@ -1864,24 +1885,18 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
|
|||
++count_dir;
|
||||
}
|
||||
|
||||
/* for each deleted block in the disk */
|
||||
first_deleted = 1;
|
||||
/* deleted blocks of the disk */
|
||||
sputsl("hole ", f);
|
||||
sputs(disk->name, f);
|
||||
sputeol(f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
for(b=0;b<blockmax;++b) {
|
||||
if (is_block_deleted(disk, b)) {
|
||||
struct snapraid_block* block = disk_block_get(disk, b);
|
||||
|
||||
if (first_deleted) {
|
||||
first_deleted = 0;
|
||||
|
||||
sputsl("hole ", f);
|
||||
sputs(disk->name, f);
|
||||
sputeol(f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
sputsl("off ", f);
|
||||
sputu32(b, f);
|
||||
sputc(' ', f);
|
||||
|
|
@ -2010,6 +2025,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
int crc_checked;
|
||||
char buffer[PATH_MAX];
|
||||
int ret;
|
||||
tommy_array disk_mapping;
|
||||
uint32_t mapping_max;
|
||||
|
||||
blockmax = 0;
|
||||
count_file = 0;
|
||||
|
|
@ -2018,6 +2035,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
count_symlink = 0;
|
||||
count_dir = 0;
|
||||
crc_checked = 0;
|
||||
mapping_max = 0;
|
||||
tommy_array_init(&disk_mapping);
|
||||
|
||||
ret = sread(f, buffer, 12);
|
||||
if (ret < 0 || memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0) {
|
||||
|
|
@ -2046,12 +2065,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
uint32_t v_idx;
|
||||
struct snapraid_file* file;
|
||||
struct snapraid_disk* disk;
|
||||
uint32_t mapping;
|
||||
|
||||
ret = sgetbs(f, buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
ret = sgetb32(f, &mapping);
|
||||
if (ret < 0 || mapping >= mapping_max) {
|
||||
decoding_error(path, f);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
disk = tommy_array_get(&disk_mapping, mapping);
|
||||
|
||||
ret = sgetb64(f, &v_size);
|
||||
if (ret < 0) {
|
||||
|
|
@ -2104,14 +2125,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* find the disk */
|
||||
disk = find_disk(state, buffer);
|
||||
if (!disk) {
|
||||
decoding_error(path, f);
|
||||
fprintf(stderr, "Disk named '%s' not found!\n", buffer);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* allocate the file */
|
||||
file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0);
|
||||
|
||||
|
|
@ -2306,20 +2319,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
/* hole */
|
||||
uint32_t v_pos;
|
||||
struct snapraid_disk* disk;
|
||||
uint32_t mapping;
|
||||
|
||||
ret = sgetbs(f, buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
ret = sgetb32(f, &mapping);
|
||||
if (ret < 0 || mapping >= mapping_max) {
|
||||
decoding_error(path, f);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* find the disk */
|
||||
disk = find_disk(state, buffer);
|
||||
if (!disk) {
|
||||
decoding_error(path, f);
|
||||
fprintf(stderr, "Disk named '%s' not found!\n", buffer);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
disk = tommy_array_get(&disk_mapping, mapping);
|
||||
|
||||
/* grow the array */
|
||||
tommy_array_grow(&disk->blockarr, blockmax);
|
||||
|
|
@ -2396,12 +2403,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
char linkto[PATH_MAX];
|
||||
struct snapraid_link* link;
|
||||
struct snapraid_disk* disk;
|
||||
uint32_t mapping;
|
||||
|
||||
ret = sgetbs(f, buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
ret = sgetb32(f, &mapping);
|
||||
if (ret < 0 || mapping >= mapping_max) {
|
||||
decoding_error(path, f);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
disk = tommy_array_get(&disk_mapping, mapping);
|
||||
|
||||
ret = sgetbs(f, sub, sizeof(sub));
|
||||
if (ret < 0) {
|
||||
|
|
@ -2420,14 +2429,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* find the disk */
|
||||
disk = find_disk(state, buffer);
|
||||
if (!disk) {
|
||||
decoding_error(path, f);
|
||||
fprintf(stderr, "Disk named '%s' not found!\n", buffer);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* allocate the link as symbolic link */
|
||||
link = link_alloc(sub, linkto, FILE_IS_SYMLINK);
|
||||
|
||||
|
|
@ -2443,12 +2444,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
char linkto[PATH_MAX];
|
||||
struct snapraid_link* link;
|
||||
struct snapraid_disk* disk;
|
||||
uint32_t mapping;
|
||||
|
||||
ret = sgetbs(f, buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
ret = sgetb32(f, &mapping);
|
||||
if (ret < 0 || mapping >= mapping_max) {
|
||||
decoding_error(path, f);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
disk = tommy_array_get(&disk_mapping, mapping);
|
||||
|
||||
ret = sgetbs(f, sub, sizeof(sub));
|
||||
if (ret < 0) {
|
||||
|
|
@ -2467,14 +2470,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* find the disk */
|
||||
disk = find_disk(state, buffer);
|
||||
if (!disk) {
|
||||
decoding_error(path, f);
|
||||
fprintf(stderr, "Disk named '%s' not found!\n", buffer);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* allocate the link as hard link */
|
||||
link = link_alloc(sub, linkto, FILE_IS_HARDLINK);
|
||||
|
||||
|
|
@ -2489,12 +2484,14 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
char sub[PATH_MAX];
|
||||
struct snapraid_dir* dir;
|
||||
struct snapraid_disk* disk;
|
||||
uint32_t mapping;
|
||||
|
||||
ret = sgetbs(f, buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
ret = sgetb32(f, &mapping);
|
||||
if (ret < 0 || mapping >= mapping_max) {
|
||||
decoding_error(path, f);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
disk = tommy_array_get(&disk_mapping, mapping);
|
||||
|
||||
ret = sgetbs(f, sub, sizeof(sub));
|
||||
if (ret < 0) {
|
||||
|
|
@ -2507,14 +2504,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* find the disk */
|
||||
disk = find_disk(state, buffer);
|
||||
if (!disk) {
|
||||
decoding_error(path, f);
|
||||
fprintf(stderr, "Disk named '%s' not found!\n", buffer);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* allocate the dir */
|
||||
dir = dir_alloc(sub);
|
||||
|
||||
|
|
@ -2595,6 +2584,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
struct snapraid_map* map;
|
||||
char uuid[UUID_MAX];
|
||||
uint32_t v_pos;
|
||||
struct snapraid_disk* disk;
|
||||
|
||||
ret = sgetbs(f, buffer, sizeof(buffer));
|
||||
if (ret < 0) {
|
||||
|
|
@ -2618,6 +2608,19 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
map = map_alloc(buffer, v_pos, uuid);
|
||||
|
||||
tommy_list_insert_tail(&state->maplist, &map->node, map);
|
||||
|
||||
/* find the disk */
|
||||
disk = find_disk(state, buffer);
|
||||
if (!disk) {
|
||||
decoding_error(path, f);
|
||||
fprintf(stderr, "Disk named '%s' not present in the configuration file!\n", buffer);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* insert in the mapping vector */
|
||||
tommy_array_grow(&disk_mapping, mapping_max + 1);
|
||||
tommy_array_set(&disk_mapping, mapping_max, disk);
|
||||
++mapping_max;
|
||||
} else if (c == 'N') {
|
||||
uint32_t crc_stored;
|
||||
uint32_t crc_computed;
|
||||
|
|
@ -2648,6 +2651,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
|
|||
}
|
||||
}
|
||||
|
||||
tommy_array_done(&disk_mapping);
|
||||
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error reading the content file '%s' at offset %"PRIi64"\n", path, stell(f));
|
||||
exit(EXIT_FAILURE);
|
||||
|
|
@ -2688,6 +2693,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
block_off_t begin;
|
||||
time_t info_oldest;
|
||||
int info_has_rehash;
|
||||
int mapping_idx;
|
||||
|
||||
count_file = 0;
|
||||
count_block = 0;
|
||||
|
|
@ -2773,16 +2779,36 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
}
|
||||
|
||||
/* for each map */
|
||||
mapping_idx = 0;
|
||||
for(i=state->maplist;i!=0;i=i->next) {
|
||||
struct snapraid_map* map = i->data;
|
||||
sputc('m', f);
|
||||
sputbs(map->name, f);
|
||||
sputb32(map->position, f);
|
||||
sputbs(map->uuid, f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
struct snapraid_disk* disk;
|
||||
|
||||
/* find the disk for this mapping */
|
||||
disk = find_disk(state, map->name);
|
||||
if (!disk) {
|
||||
fprintf(stderr, "Internal incosistency for unmapped disk '%s'\n", map->name);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* save the mapping only for not empty disks */
|
||||
if (!disk_is_empty(disk, blockmax)) {
|
||||
sputc('m', f);
|
||||
sputbs(map->name, f);
|
||||
sputb32(map->position, f);
|
||||
sputbs(map->uuid, f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
/* assign the mapping index used to identify disks */
|
||||
disk->mapping = mapping_idx;
|
||||
++mapping_idx;
|
||||
} else {
|
||||
/* mark the disk as without mapping */
|
||||
disk->mapping = -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* for each disk */
|
||||
|
|
@ -2790,6 +2816,10 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
tommy_node* j;
|
||||
struct snapraid_disk* disk = i->data;
|
||||
|
||||
/* if the disk is not mapped, skip it */
|
||||
if (disk->mapping < 0)
|
||||
continue;
|
||||
|
||||
/* for each file */
|
||||
for(j=disk->filelist;j!=0;j=j->next) {
|
||||
struct snapraid_file* file = j->data;
|
||||
|
|
@ -2805,7 +2835,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
inode = file->inode;
|
||||
|
||||
sputc('f', f);
|
||||
sputbs(disk->name, f);
|
||||
sputb32(disk->mapping, f);
|
||||
sputb64(size, f);
|
||||
sputb64(mtime_sec, f);
|
||||
/* encode STAT_NSEC_INVALID as 0 */
|
||||
|
|
@ -2893,7 +2923,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
break;
|
||||
}
|
||||
|
||||
sputbs(disk->name, f);
|
||||
sputb32(disk->mapping, f);
|
||||
sputbs(link->sub, f);
|
||||
sputbs(link->linkto, f);
|
||||
if (serror(f)) {
|
||||
|
|
@ -2907,7 +2937,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
struct snapraid_dir* dir = j->data;
|
||||
|
||||
sputc('r', f);
|
||||
sputbs(disk->name, f);
|
||||
sputb32(disk->mapping, f);
|
||||
sputbs(dir->sub, f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
|
|
@ -2917,7 +2947,13 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
++count_dir;
|
||||
}
|
||||
|
||||
/* write the deleted info for each block */
|
||||
/* deleted blocks of the disk */
|
||||
sputc('h', f);
|
||||
sputb32(disk->mapping, f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
begin = 0;
|
||||
while (begin < blockmax) {
|
||||
int is_deleted;
|
||||
|
|
@ -2933,24 +2969,6 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
|
|||
++end;
|
||||
}
|
||||
|
||||
/* if it's the first iteration */
|
||||
if (begin == 0) {
|
||||
/* if there is no deleted block in the disk, write nothing */
|
||||
/* this is required to avoid to have references to a completely empty disk */
|
||||
if (end == blockmax && !is_deleted) {
|
||||
/* exit */
|
||||
break;
|
||||
}
|
||||
|
||||
/* write the hole header only if required */
|
||||
sputc('h', f);
|
||||
sputbs(disk->name, f);
|
||||
if (serror(f)) {
|
||||
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
sputb32(end - begin, f);
|
||||
|
||||
if (is_deleted) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue