From f4f55542876fb35fe589fe1cc5c5861556e1f7ef Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 28 Apr 2011 23:09:19 +0200 Subject: [PATCH] Reduces memory occupation. --- TODO | 2 -- check.c | 32 +++++++++++++++--------------- dry.c | 8 ++++---- elem.c | 16 +++++++-------- elem.h | 61 +++++++++++++++++++++++++++++++++++++++++++++++---------- scan.c | 10 +++++----- state.c | 16 ++++++++------- sync.c | 20 +++++++++---------- util.h | 18 ----------------- 9 files changed, 102 insertions(+), 81 deletions(-) diff --git a/TODO b/TODO index 79cf6549..b42dd060 100644 --- a/TODO +++ b/TODO @@ -10,8 +10,6 @@ This is a list of possible TODO for SnapRAID. The order is not relevant. * When fixing, overwrite the present file only if the backuped one can be completely restored. -* Reduce memory occupation removing the flag field in the block structure. - * Recognize that a file is moved from one disk to another, and if the parity data doesn't overlap, do not recompute it. diff --git a/check.c b/check.c index ff93e17e..03912e53 100644 --- a/check.c +++ b/check.c @@ -204,8 +204,8 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity for(j=0;jflag, BLOCK_HAS_HASH) /* only if the block is hashed */ - && !block->file->is_excluded /* only if the file is not filtered out */ + && block_flag_has(block, BLOCK_HAS_HASH) /* only if the block is hashed */ + && !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */ ) { one_tocheck = 1; break; @@ -234,8 +234,8 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity for(j=0;jflag, BLOCK_HAS_HASH) /* only if the block is hashed */ - && !block->file->is_excluded /* only if the file is not filtered out */ + && block_flag_has(block, BLOCK_HAS_HASH) /* only if the block is hashed */ + && !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */ ) { one_tocheck = 1; break; @@ -265,17 +265,17 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity /* we try to check and fix only if the block is hashed */ /* it could be that a file was added, and not yet synched */ /* so we should not include not hashed block in the parity computation */ - if (!bit_has(block->flag, BLOCK_HAS_HASH)) { + if (!block_flag_has(block, BLOCK_HAS_HASH)) { /* use an empty block */ memset(buffer[j], 0, state->block_size); continue; } /* keep track if at least one hashed block has no parity */ - if (!bit_has(block->flag, BLOCK_HAS_PARITY)) + if (!block_flag_has(block, BLOCK_HAS_PARITY)) all_parity = 0; - ret = handle_close_if_different(&handle[j], block->file); + ret = handle_close_if_different(&handle[j], block_file_get(block)); if (ret == -1) { fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); fprintf(stderr, "Stopping at block %u\n", i); @@ -285,7 +285,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity if (fix) { /* if fixing, create the file, open for writing and resize if required */ - ret = handle_create(&handle[j], block->file); + ret = handle_create(&handle[j], block_file_get(block)); if (ret == -1) { fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); fprintf(stderr, "Stopping at block %u\n", i); @@ -301,7 +301,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } } else { /* if checking, open the file for reading */ - ret = handle_open(&handle[j], block->file); + ret = handle_open(&handle[j], block_file_get(block)); if (ret == -1) { /* save the failed block for the parity check */ if (failed_count < 2) { @@ -311,17 +311,17 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } ++failed_count; - fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); + fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block)); ++error; continue; } /* check if it's a larger file, but not if already notified */ - if (!block->file->is_larger && handle[j].st.st_size > block->file->size) { + if (!file_flag_has(block_file_get(block), FILE_IS_LARGER) && handle[j].st.st_size > block_file_get(block)->size) { fprintf(stderr, "File '%s' is larger than expected.\n", handle->path); /* if fragmented, it may be reopened, so store the notification */ - block->file->is_larger = 1; + file_flag_set(block_file_get(block), FILE_IS_LARGER); /* this is always a recoverable error */ ++error; @@ -338,7 +338,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } ++failed_count; - fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); + fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block)); ++error; continue; } @@ -356,7 +356,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } ++failed_count; - fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); + fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block)); ++error; continue; } @@ -438,7 +438,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity /* update the fixed files */ for(j=0;jfile->is_excluded) + if (file_flag_has(block_file_get(failed[j].block), FILE_IS_EXCLUDED)) continue; ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size); @@ -449,7 +449,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity goto bail; } - fprintf(stderr, "%u: Fixed data error for file %s at position %u\n", i, failed[j].block->file->sub, block_file_pos(failed[j].block)); + fprintf(stderr, "%u: Fixed data error for file %s at position %u\n", i, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block)); ++recovered_error; } diff --git a/dry.c b/dry.c index 59f1fb74..04263181 100644 --- a/dry.c +++ b/dry.c @@ -66,7 +66,7 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, int qar if (!block) continue; - ret = handle_close_if_different(&handle[j], block->file); + ret = handle_close_if_different(&handle[j], block_file_get(block)); if (ret == -1) { fprintf(stderr, "Stopping at block %u\n", i); ++error; @@ -74,16 +74,16 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, int qar } /* open the file for reading */ - ret = handle_open(&handle[j], block->file); + ret = handle_open(&handle[j], block_file_get(block)); if (ret == -1) { - fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); + fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block)); ++error; continue; } read_size = handle_read(&handle[j], block, buffer, state->block_size); if (read_size == -1) { - fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); + fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block)); ++error; continue; } diff --git a/elem.c b/elem.c index ed92c5a1..8266472e 100644 --- a/elem.c +++ b/elem.c @@ -183,7 +183,7 @@ int filter_path(tommy_list* filterlist, const char* path, int is_dir) block_off_t block_file_pos(struct snapraid_block* block) { - struct snapraid_file* file = block->file; + struct snapraid_file* file = block_file_get(block); if (block < file->blockvec || block >= file->blockvec + file->blockmax) { fprintf(stderr, "Internal inconsistency in block %u ownership\n", block->parity_pos); @@ -198,11 +198,11 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size) block_off_t pos = block_file_pos(block); /* if it's the last block */ - if (pos + 1 == block->file->blockmax) { + if (pos + 1 == block_file_get(block)->blockmax) { unsigned remainder; - if (block->file->size == 0) + if (block_file_get(block)->size == 0) return 0; - remainder = block->file->size % block_size; + remainder = block_file_get(block)->size % block_size; if (remainder == 0) remainder = block_size; return remainder; @@ -222,16 +222,14 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t file->blockmax = (size + block_size - 1) / block_size; file->mtime = mtime; file->inode = inode; - file->is_present = 0; - file->is_excluded = 0; - file->is_larger = 0; + file->flag = 0; file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); /* set the back pointer */ for(i=0;iblockmax;++i) { file->blockvec[i].parity_pos = POS_INVALID; - file->blockvec[i].flag = 0; - file->blockvec[i].file = file; + file->blockvec[i].file_mixed = 0; + block_file_set(&file->blockvec[i], file); } return file; diff --git a/elem.h b/elem.h index 30758910..d30e951e 100644 --- a/elem.h +++ b/elem.h @@ -1,4 +1,4 @@ -/* +0/* * Copyright (C) 2011 Andrea Mazzoleni * * This program is free software: you can redistribute it and/or modify @@ -73,24 +73,26 @@ struct snapraid_file; */ struct snapraid_block { block_off_t parity_pos; /**< Position of the block in the parity. */ - unsigned flag; /**< If the hash of the block is valid. */ - struct snapraid_file* file; /**< Back pointer to the file owning this block. */ + uintptr_t file_mixed; /**< Back pointer to the file owning this block, mixed with some flags. */ unsigned char hash[HASH_SIZE]; /**< Hash of the block. */ }; +#define FILE_IS_PRESENT 1 /**< If it's seen as present. */ +#define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */ +#define FILE_IS_LARGER 4 /**< If a larger file was already detected. */ + /** * File. */ struct snapraid_file { - char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */ + int64_t mtime; /**< Modification time. */ + uint64_t inode; /**< Inode. */ data_off_t size; /**< Size of the file. */ struct snapraid_block* blockvec; /**< All the blocks of the file. */ block_off_t blockmax; /**< Number of blocks. */ - int64_t mtime; /**< Modification time. */ - uint64_t inode; /**< Inode. */ - int is_present; /**< If it's seen as present. */ - int is_excluded; /**< If it's an excluded file from the processing. */ - int is_larger; /**< If a larger file was already detected. */ + unsigned flag; /**< FILE_IS_* flags. */ + char sub[PATH_MAX]; /**< Sub path of the file. Without the disk dir. The disk is implicit. */ + /* nodes for data structures */ tommy_node nodelist; tommy_hashdyn_node nodeset; @@ -128,7 +130,6 @@ struct snapraid_filter* filter_alloc(int is_include, const char* pattern); */ void filter_free(struct snapraid_filter* filter); - /** * Filters a path. * For each element of the path all the filters are applied, until the first one that matches. @@ -136,6 +137,31 @@ void filter_free(struct snapraid_filter* filter); */ int filter_path(tommy_list* filterlist, const char* path, int is_dir); +static inline struct snapraid_file* block_file_get(struct snapraid_block* block) +{ + return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)3); +} + +static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file) +{ + block->file_mixed = (block->file_mixed & ~(uintptr_t)3) | (uintptr_t)file; +} + +static inline int block_flag_has(struct snapraid_block* block, unsigned mask) +{ + return (block->file_mixed & mask) == mask; +} + +static inline void block_flag_set(struct snapraid_block* block, unsigned mask) +{ + block->file_mixed |= mask; +} + +static inline void block_flag_clear(struct snapraid_block* block, unsigned mask) +{ + block->file_mixed &= ~(uintptr_t)mask; +} + /** * Gets the relative position of a block inside the file. */ @@ -147,6 +173,21 @@ block_off_t block_file_pos(struct snapraid_block* block); */ unsigned block_file_size(struct snapraid_block* block, unsigned block_size); +static inline int file_flag_has(struct snapraid_file* file, unsigned mask) +{ + return (file->flag & mask) == mask; +} + +static inline void file_flag_set(struct snapraid_file* file, unsigned mask) +{ + file->flag |= mask; +} + +static inline void file_flag_clear(struct snapraid_file* file, unsigned mask) +{ + file->flag &= ~mask; +} + /** * Allocates a file. */ diff --git a/scan.c b/scan.c index a9059abb..df5d65a6 100644 --- a/scan.c +++ b/scan.c @@ -57,7 +57,7 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos); if (oth_block) { /* remove the parity info for this block */ - oth_block->flag = bit_clear(oth_block->flag, BLOCK_HAS_PARITY); + block_flag_clear(oth_block, BLOCK_HAS_PARITY); } } } @@ -123,7 +123,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare, &inode, file_inode_hash(inode)); if (file) { /* check if multiple files have the same inode */ - if (file->is_present) { + if (file_flag_has(file, FILE_IS_PRESENT)) { if (st->st_nlink > 1) { printf("warning: Ignored hardlink '/%s'\n", sub); return; @@ -136,7 +136,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, /* check if the file is not changed */ if (file->size == st->st_size && file->mtime == st->st_mtime) { /* mark as present */ - file->is_present = 1; + file_flag_set(file, FILE_IS_PRESENT); /* if the path is different, it means a moved file */ if (strcmp(file->sub, sub) != 0) { @@ -180,7 +180,7 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, file = file_alloc(state->block_size, sub, st->st_size, st->st_mtime, st->st_ino); /* mark it as present */ - file->is_present = 1; + file_flag_set(file, FILE_IS_PRESENT); /* insert it */ scan_file_insert(state, disk, file); @@ -284,7 +284,7 @@ void state_scan(struct snapraid_state* state) node = node->next; /* remove if not present */ - if (!file->is_present) { + if (!file_flag_has(file, FILE_IS_PRESENT)) { scan_file_remove(state, disk, file); ++scan[i].count_remove; } diff --git a/state.c b/state.c index 1877e6a2..d296a1c4 100644 --- a/state.c +++ b/state.c @@ -293,15 +293,15 @@ void state_read(struct snapraid_state* state) exit(EXIT_FAILURE); } - block->flag = bit_set(block->flag, BLOCK_HAS_HASH); + block_flag_set(block, BLOCK_HAS_HASH); } /* set the parity only if present */ if (tag[0] == 'b') /* blk */ - block->flag = bit_set(block->flag, BLOCK_HAS_PARITY); + block_flag_set(block, BLOCK_HAS_PARITY); /* parity implies hash */ - if (bit_has(block->flag, BLOCK_HAS_PARITY) && !bit_has(block->flag, BLOCK_HAS_HASH)) { + if (block_flag_has(block, BLOCK_HAS_PARITY) && !block_flag_has(block, BLOCK_HAS_HASH)) { fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } @@ -500,13 +500,13 @@ static void state_write_one(struct snapraid_state* state, const char* path, unsi struct snapraid_block* block = &file->blockvec[k]; const char* tag; - if (bit_has(block->flag, BLOCK_HAS_PARITY)) { + if (block_flag_has(block, BLOCK_HAS_PARITY)) { tag = "blk"; } else { tag = "inv"; } - if (bit_has(block->flag, BLOCK_HAS_HASH)) { + if (block_flag_has(block, BLOCK_HAS_HASH)) { char s_hash[HASH_SIZE*2+1]; strenchex(s_hash, block->hash, HASH_SIZE); s_hash[HASH_SIZE*2] = 0; @@ -606,9 +606,11 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist) for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) { struct snapraid_file* file = j->data; - file->is_excluded = filter_path(filterlist, file->sub, 0) != 0; + if (filter_path(filterlist, file->sub, 0) != 0) { + file_flag_set(file, FILE_IS_EXCLUDED); + } - if (state->verbose && !file->is_excluded) { + if (state->verbose && !file_flag_has(file, FILE_IS_EXCLUDED)) { printf("Processing file '%s'\n", file->sub); } } diff --git a/sync.c b/sync.c index b7ec3f0d..d43e6270 100644 --- a/sync.c +++ b/sync.c @@ -71,7 +71,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa one_invalid = 0; for(j=0;jflag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) { + if (block && !block_flag_has(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) { one_invalid = 1; break; } @@ -96,7 +96,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa one_invalid = 0; for(j=0;jflag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) { + if (block && !block_flag_has(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) { one_invalid = 1; break; } @@ -119,7 +119,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa continue; } - ret = handle_close_if_different(&handle[j], block->file); + ret = handle_close_if_different(&handle[j], block_file_get(block)); if (ret == -1) { /* This one is really an unexpected error, because we are only reading */ /* and closing a descriptor should never fail */ @@ -129,7 +129,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa goto bail; } - ret = handle_open(&handle[j], block->file); + ret = handle_open(&handle[j], block_file_get(block)); if (ret == -1) { if (errno == ENOENT) { fprintf(stderr, "Missing file '%s'.\n", handle[j].path); @@ -148,9 +148,9 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa } /* check if the file is changed */ - if (handle[j].st.st_size != block->file->size - || handle[j].st.st_mtime != block->file->mtime - || handle[j].st.st_ino != block->file->inode + if (handle[j].st.st_size != block_file_get(block)->size + || handle[j].st.st_mtime != block_file_get(block)->mtime + || handle[j].st.st_ino != block_file_get(block)->inode ) { fprintf(stderr, "Unexpected change at file '%s'.\n", handle[j].path); fprintf(stderr, "WARNING! You cannot modify data disk during a sync. Rerun the sync command when finished.\n"); @@ -170,10 +170,10 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa /* now compute the hash */ memhash(state->hash, hash, buffer[j], read_size); - if (bit_has(block->flag, BLOCK_HAS_HASH)) { + if (block_flag_has(block, BLOCK_HAS_HASH)) { /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { - fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); + fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block)); fprintf(stderr, "DANGER! Unexpected data error in a data disk, it isn't possible to sync.\n"); fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid -s %u check'\n", i); ++unrecoverable_error; @@ -219,7 +219,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, int qa if (!block) continue; - block->flag = bit_set(block->flag, BLOCK_HAS_HASH | BLOCK_HAS_PARITY); + block_flag_set(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY); } /* mark the state as needing write */ diff --git a/util.h b/util.h index 703952c6..71230fe6 100644 --- a/util.h +++ b/util.h @@ -94,24 +94,6 @@ void pathprint(char* str, size_t size, const char* format, ...); */ void pathslash(char* str, size_t size); -/****************************************************************************/ -/* bit */ - -static inline int bit_has(unsigned bit, unsigned mask) -{ - return (bit & mask) == mask; -} - -static inline unsigned bit_set(unsigned bit, unsigned mask) -{ - return bit | mask; -} - -static inline unsigned bit_clear(unsigned bit, unsigned mask) -{ - return bit & ~mask; -} - /****************************************************************************/ /* memory */