rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reduces memory occupation.

Browse source
This commit is contained in:
Andrea Mazzoleni 2011-04-28 23:09:19 +02:00
parent 19d660f383
commit f4f5554287
9 changed files with 102 additions and 81 deletions
Download patch file Download diff file Expand all files Collapse all files

2
TODO
View file

@ -10,8 +10,6 @@ This is a list of possible TODO for SnapRAID. The order is not relevant.
* When fixing, overwrite the present file only if the backuped one can be
completely restored.
* Reduce memory occupation removing the flag field in the block structure.
* Recognize that a file is moved from one disk to another, and if the parity
data doesn't overlap, do not recompute it.

32
check.c
View file

@ -204,8 +204,8 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block
&& bit_has(block->flag, BLOCK_HAS_HASH) /* only if the block is hashed */
&& !block->file->is_excluded /* only if the file is not filtered out */
&& block_flag_has(block, BLOCK_HAS_HASH) /* only if the block is hashed */
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
break;
@ -234,8 +234,8 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block
&& bit_has(block->flag, BLOCK_HAS_HASH) /* only if the block is hashed */
&& !block->file->is_excluded /* only if the file is not filtered out */
&& block_flag_has(block, BLOCK_HAS_HASH) /* only if the block is hashed */
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
break;
@ -265,17 +265,17 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
/* we try to check and fix only if the block is hashed */
/* it could be that a file was added, and not yet synched */
/* so we should not include not hashed block in the parity computation */
if (!bit_has(block->flag, BLOCK_HAS_HASH)) {
if (!block_flag_has(block, BLOCK_HAS_HASH)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* keep track if at least one hashed block has no parity */
if (!bit_has(block->flag, BLOCK_HAS_PARITY))
if (!block_flag_has(block, BLOCK_HAS_PARITY))
all_parity = 0;
ret = handle_close_if_different(&handle[j], block->file);
ret = handle_close_if_different(&handle[j], block_file_get(block));
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
@ -285,7 +285,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
if (fix) {
/* if fixing, create the file, open for writing and resize if required */
ret = handle_create(&handle[j], block->file);
ret = handle_create(&handle[j], block_file_get(block));
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
@ -301,7 +301,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
} else {
/* if checking, open the file for reading */
ret = handle_open(&handle[j], block->file);
ret = handle_open(&handle[j], block_file_get(block));
if (ret == -1) {
/* save the failed block for the parity check */
if (failed_count < 2) {
@ -311,17 +311,17 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
++failed_count;
fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
/* check if it's a larger file, but not if already notified */
if (!block->file->is_larger && handle[j].st.st_size > block->file->size) {
if (!file_flag_has(block_file_get(block), FILE_IS_LARGER) && handle[j].st.st_size > block_file_get(block)->size) {
fprintf(stderr, "File '%s' is larger than expected.\n", handle->path);
/* if fragmented, it may be reopened, so store the notification */
block->file->is_larger = 1;
file_flag_set(block_file_get(block), FILE_IS_LARGER);
/* this is always a recoverable error */
++error;
@ -338,7 +338,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
++failed_count;
fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
@ -356,7 +356,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
++failed_count;
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
@ -438,7 +438,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
/* update the fixed files */
for(j=0;j<failed_count;++j) {
/* do not fix if the file filtered out */
if (failed[j].block->file->is_excluded)
if (file_flag_has(block_file_get(failed[j].block), FILE_IS_EXCLUDED))
continue;
ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size);
@ -449,7 +449,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
goto bail;
}
fprintf(stderr, "%u: Fixed data error for file %s at position %u\n", i, failed[j].block->file->sub, block_file_pos(failed[j].block));
fprintf(stderr, "%u: Fixed data error for file %s at position %u\n", i, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
++recovered_error;
}

8
dry.c
View file

@ -66,7 +66,7 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, int qar
if (!block)
continue;
ret = handle_close_if_different(&handle[j], block->file);
ret = handle_close_if_different(&handle[j], block_file_get(block));
if (ret == -1) {
fprintf(stderr, "Stopping at block %u\n", i);
++error;
@ -74,16 +74,16 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, int qar
}
/* open the file for reading */
ret = handle_open(&handle[j], block->file);
ret = handle_open(&handle[j], block_file_get(block));
if (ret == -1) {
fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
read_size = handle_read(&handle[j], block, buffer, state->block_size);
if (read_size == -1) {
fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}

16
elem.c
View file

@ -183,7 +183,7 @@ int filter_path(tommy_list* filterlist, const char* path, int is_dir)
block_off_t block_file_pos(struct snapraid_block* block)
{
struct snapraid_file* file = block->file;
struct snapraid_file* file = block_file_get(block);
if (block < file->blockvec || block >= file->blockvec + file->blockmax) {
fprintf(stderr, "Internal inconsistency in block %u ownership\n", block->parity_pos);
@ -198,11 +198,11 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size)
block_off_t pos = block_file_pos(block);
/* if it's the last block */
if (pos + 1 == block->file->blockmax) {
if (pos + 1 == block_file_get(block)->blockmax) {
unsigned remainder;
if (block->file->size == 0)
if (block_file_get(block)->size == 0)
return 0;
remainder = block->file->size % block_size;
remainder = block_file_get(block)->size % block_size;
if (remainder == 0)
remainder = block_size;
return remainder;
@ -222,16 +222,14 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t
file->blockmax = (size + block_size - 1) / block_size;
file->mtime = mtime;
file->inode = inode;
file->is_present = 0;
file->is_excluded = 0;
file->is_larger = 0;
file->flag = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
/* set the back pointer */
for(i=0;i<file->blockmax;++i) {
file->blockvec[i].parity_pos = POS_INVALID;
file->blockvec[i].flag = 0;
file->blockvec[i].file = file;
file->blockvec[i].file_mixed = 0;
block_file_set(&file->blockvec[i], file);
}
return file;

61
elem.h
View file

@ -1,4 +1,4 @@
/*
0/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
@ -73,24 +73,26 @@ struct snapraid_file;
*/
struct snapraid_block {
block_off_t parity_pos; /**< Position of the block in the parity. */
unsigned flag; /**< If the hash of the block is valid. */
struct snapraid_file* file; /**< Back pointer to the file owning this block. */
uintptr_t file_mixed; /**< Back pointer to the file owning this block, mixed with some flags. */
unsigned char hash[HASH_SIZE]; /**< Hash of the block. */
};
#define FILE_IS_PRESENT 1 /**< If it's seen as present. */
#define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */
#define FILE_IS_LARGER 4 /**< If a larger file was already detected. */
/**
* File.
*/
struct snapraid_file {
char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
int64_t mtime; /**< Modification time. */
uint64_t inode; /**< Inode. */
data_off_t size; /**< Size of the file. */
struct snapraid_block* blockvec; /**< All the blocks of the file. */
block_off_t blockmax; /**< Number of blocks. */
int64_t mtime; /**< Modification time. */
uint64_t inode; /**< Inode. */
int is_present; /**< If it's seen as present. */
int is_excluded; /**< If it's an excluded file from the processing. */
int is_larger; /**< If a larger file was already detected. */
unsigned flag; /**< FILE_IS_* flags. */
char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */
/* nodes for data structures */
tommy_node nodelist;
tommy_hashdyn_node nodeset;
@ -128,7 +130,6 @@ struct snapraid_filter* filter_alloc(int is_include, const char* pattern);
*/
void filter_free(struct snapraid_filter* filter);
/**
* Filters a path.
* For each element of the path all the filters are applied, until the first one that matches.
@ -136,6 +137,31 @@ void filter_free(struct snapraid_filter* filter);
*/
int filter_path(tommy_list* filterlist, const char* path, int is_dir);
static inline struct snapraid_file* block_file_get(struct snapraid_block* block)
{
return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)3);
}
static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file)
{
block->file_mixed = (block->file_mixed & ~(uintptr_t)3) | (uintptr_t)file;
}
static inline int block_flag_has(struct snapraid_block* block, unsigned mask)
{
return (block->file_mixed & mask) == mask;
}
static inline void block_flag_set(struct snapraid_block* block, unsigned mask)
{
block->file_mixed |= mask;
}
static inline void block_flag_clear(struct snapraid_block* block, unsigned mask)
{
block->file_mixed &= ~(uintptr_t)mask;
}
/**
* Gets the relative position of a block inside the file.
*/
@ -147,6 +173,21 @@ block_off_t block_file_pos(struct snapraid_block* block);
*/
unsigned block_file_size(struct snapraid_block* block, unsigned block_size);
static inline int file_flag_has(struct snapraid_file* file, unsigned mask)
{
return (file->flag & mask) == mask;
}
static inline void file_flag_set(struct snapraid_file* file, unsigned mask)
{
file->flag |= mask;
}
static inline void file_flag_clear(struct snapraid_file* file, unsigned mask)
{
file->flag &= ~mask;
}
/**
* Allocates a file.
*/

10
scan.c
View file

@ -57,7 +57,7 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
if (oth_block) {
/* remove the parity info for this block */
oth_block->flag = bit_clear(oth_block->flag, BLOCK_HAS_PARITY);
block_flag_clear(oth_block, BLOCK_HAS_PARITY);
}
}
}
@ -123,7 +123,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare, &inode, file_inode_hash(inode));
if (file) {
/* check if multiple files have the same inode */
if (file->is_present) {
if (file_flag_has(file, FILE_IS_PRESENT)) {
if (st->st_nlink > 1) {
printf("warning: Ignored hardlink '/%s'\n", sub);
return;
@ -136,7 +136,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
/* check if the file is not changed */
if (file->size == st->st_size && file->mtime == st->st_mtime) {
/* mark as present */
file->is_present = 1;
file_flag_set(file, FILE_IS_PRESENT);
/* if the path is different, it means a moved file */
if (strcmp(file->sub, sub) != 0) {
@ -180,7 +180,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, st->st_ino);
/* mark it as present */
file->is_present = 1;
file_flag_set(file, FILE_IS_PRESENT);
/* insert it */
scan_file_insert(state, disk, file);
@ -284,7 +284,7 @@ void state_scan(struct snapraid_state* state)
node = node->next;
/* remove if not present */
if (!file->is_present) {
if (!file_flag_has(file, FILE_IS_PRESENT)) {
scan_file_remove(state, disk, file);
++scan[i].count_remove;
}

16
state.c
View file

@ -293,15 +293,15 @@ void state_read(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
block->flag = bit_set(block->flag, BLOCK_HAS_HASH);
block_flag_set(block, BLOCK_HAS_HASH);
}
/* set the parity only if present */
if (tag[0] == 'b') /* blk */
block->flag = bit_set(block->flag, BLOCK_HAS_PARITY);
block_flag_set(block, BLOCK_HAS_PARITY);
/* parity implies hash */
if (bit_has(block->flag, BLOCK_HAS_PARITY) && !bit_has(block->flag, BLOCK_HAS_HASH)) {
if (block_flag_has(block, BLOCK_HAS_PARITY) && !block_flag_has(block, BLOCK_HAS_HASH)) {
fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
@ -500,13 +500,13 @@ static void state_write_one(struct snapraid_state* state, const char* path, unsi
struct snapraid_block* block = &file->blockvec[k];
const char* tag;
if (bit_has(block->flag, BLOCK_HAS_PARITY)) {
if (block_flag_has(block, BLOCK_HAS_PARITY)) {
tag = "blk";
} else {
tag = "inv";
}
if (bit_has(block->flag, BLOCK_HAS_HASH)) {
if (block_flag_has(block, BLOCK_HAS_HASH)) {
char s_hash[HASH_SIZE*2+1];
strenchex(s_hash, block->hash, HASH_SIZE);
s_hash[HASH_SIZE*2] = 0;
@ -606,9 +606,11 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist)
for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) {
struct snapraid_file* file = j->data;
file->is_excluded = filter_path(filterlist, file->sub, 0) != 0;
if (filter_path(filterlist, file->sub, 0) != 0) {
file_flag_set(file, FILE_IS_EXCLUDED);
}
if (state->verbose && !file->is_excluded) {
if (state->verbose && !file_flag_has(file, FILE_IS_EXCLUDED)) {
printf("Processing file '%s'\n", file->sub);
}
}

20
sync.c
View file

@ -71,7 +71,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
one_invalid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block && !bit_has(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
if (block && !block_flag_has(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
one_invalid = 1;
break;
}
@ -96,7 +96,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
one_invalid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block && !bit_has(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
if (block && !block_flag_has(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
one_invalid = 1;
break;
}
@ -119,7 +119,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
continue;
}
ret = handle_close_if_different(&handle[j], block->file);
ret = handle_close_if_different(&handle[j], block_file_get(block));
if (ret == -1) {
/* This one is really an unexpected error, because we are only reading */
/* and closing a descriptor should never fail */
@ -129,7 +129,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
goto bail;
}
ret = handle_open(&handle[j], block->file);
ret = handle_open(&handle[j], block_file_get(block));
if (ret == -1) {
if (errno == ENOENT) {
fprintf(stderr, "Missing file '%s'.\n", handle[j].path);
@ -148,9 +148,9 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
}
/* check if the file is changed */
if (handle[j].st.st_size != block->file->size
|| handle[j].st.st_mtime != block->file->mtime
|| handle[j].st.st_ino != block->file->inode
if (handle[j].st.st_size != block_file_get(block)->size
|| handle[j].st.st_mtime != block_file_get(block)->mtime
|| handle[j].st.st_ino != block_file_get(block)->inode
) {
fprintf(stderr, "Unexpected change at file '%s'.\n", handle[j].path);
fprintf(stderr, "WARNING! You cannot modify data disk during a sync. Rerun the sync command when finished.\n");
@ -170,10 +170,10 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
/* now compute the hash */
memhash(state->hash, hash, buffer[j], read_size);
if (bit_has(block->flag, BLOCK_HAS_HASH)) {
if (block_flag_has(block, BLOCK_HAS_HASH)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
fprintf(stderr, "DANGER! Unexpected data error in a data disk, it isn't possible to sync.\n");
fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u check'\n", i);
++unrecoverable_error;
@ -219,7 +219,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa
if (!block)
continue;
block->flag = bit_set(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY);
block_flag_set(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY);
}
/* mark the state as needing write */

18
util.h
View file

@ -94,24 +94,6 @@ void pathprint(char* str, size_t size, const char* format, ...);
*/
void pathslash(char* str, size_t size);
/****************************************************************************/
/* bit */
static inline int bit_has(unsigned bit, unsigned mask)
{
return (bit & mask) == mask;
}
static inline unsigned bit_set(unsigned bit, unsigned mask)
{
return bit | mask;
}
static inline unsigned bit_clear(unsigned bit, unsigned mask)
{
return bit & ~mask;
}
/****************************************************************************/
/* memory */