mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 02:49:02 -04:00
Split scrub disk loop to support batched hashing
This commit is contained in:
Andrea Mazzoleni
parent
1dd3439fe8
commit
13844afe4c
1 changed files with 79 additions and 49 deletions
128
cmdline/scrub.c
128
cmdline/scrub.c
|
|
@ -23,11 +23,15 @@ static const char* es(int err)
|
|||
}
|
||||
|
||||
/**
|
||||
* Buffer for storing the new hashes.
|
||||
* Buffer for saving state between loops.
|
||||
*/
|
||||
struct snapraid_rehash {
|
||||
struct snapraid_disk_state {
|
||||
struct snapraid_task* task;
|
||||
unsigned char hash[HASH_MAX];
|
||||
struct snapraid_block* block;
|
||||
unsigned char rehash[HASH_MAX];
|
||||
unsigned diskcur;
|
||||
int needs_hash;
|
||||
int file_is_unsynced;
|
||||
};
|
||||
|
||||
/**
|
||||
|
|
@ -239,8 +243,11 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
{
|
||||
struct snapraid_io io;
|
||||
struct snapraid_handle* handle;
|
||||
void* rehandle_alloc;
|
||||
struct snapraid_rehash* rehandle;
|
||||
struct snapraid_disk_state* disk_state;
|
||||
void** hash_digest;
|
||||
void** rehash_digest;
|
||||
const void** hash_src;
|
||||
size_t* hash_size;
|
||||
unsigned diskmax;
|
||||
block_off_t blockcur;
|
||||
unsigned j;
|
||||
|
|
@ -264,8 +271,12 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
/* maps the disks to handles */
|
||||
handle = handle_mapping(state, &diskmax);
|
||||
|
||||
/* rehash buffers */
|
||||
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
|
||||
/* allocate snapraid_disk_state and batch arrays */
|
||||
disk_state = malloc_nofail(diskmax * sizeof(struct snapraid_disk_state));
|
||||
hash_digest = malloc_nofail(diskmax * sizeof(void*));
|
||||
rehash_digest = malloc_nofail(diskmax * sizeof(void*));
|
||||
hash_src = malloc_nofail(diskmax * sizeof(const void*));
|
||||
hash_size = malloc_nofail(diskmax * sizeof(size_t));
|
||||
|
||||
/* we need 1 * data + 2 * parity */
|
||||
buffermax = diskmax + 2 * state->level;
|
||||
|
|
@ -328,7 +339,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
int silent_error_on_this_block;
|
||||
int io_error_on_this_block;
|
||||
int block_is_unsynced;
|
||||
int rehash;
|
||||
int needs_rehash;
|
||||
void** buffer;
|
||||
|
||||
/* go to the next block */
|
||||
|
|
@ -358,44 +369,37 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
info = info_get(&state->infoarr, blockcur);
|
||||
|
||||
/* if we have to use the old hash */
|
||||
rehash = info_get_rehash(info);
|
||||
needs_rehash = info_get_rehash(info);
|
||||
|
||||
/* for each disk, process the block */
|
||||
/* clear disk_state needs_hash flags */
|
||||
for (j = 0; j < diskmax; ++j) {
|
||||
struct snapraid_task* task;
|
||||
ssize_t read_size;
|
||||
unsigned char hash[HASH_MAX];
|
||||
struct snapraid_block* block;
|
||||
int file_is_unsynced;
|
||||
struct snapraid_disk* disk;
|
||||
struct snapraid_file* file;
|
||||
block_off_t file_pos;
|
||||
unsigned diskcur;
|
||||
disk_state[j].needs_hash = 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* If the file on this disk is synced
|
||||
* if not, silent errors are assumed as expected error
|
||||
*/
|
||||
file_is_unsynced = 0;
|
||||
unsigned hash_count = 0;
|
||||
|
||||
/* for each disk, read the block */
|
||||
for (j = 0; j < diskmax; ++j) {
|
||||
unsigned diskcur;
|
||||
struct snapraid_disk_state* ds;
|
||||
|
||||
/* until now is misc */
|
||||
state_usage_misc(state);
|
||||
|
||||
/* get the next task */
|
||||
task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
|
||||
struct snapraid_task* task = io_data_read(&io, &diskcur, waiting_map, &waiting_mac);
|
||||
struct snapraid_block* block = task->block;
|
||||
struct snapraid_disk* disk = task->disk;
|
||||
struct snapraid_file* file = task->file;
|
||||
|
||||
/* until now is disk */
|
||||
state_usage_disk(state, handle, waiting_map, waiting_mac);
|
||||
|
||||
/* get the task results */
|
||||
disk = task->disk;
|
||||
block = task->block;
|
||||
file = task->file;
|
||||
file_pos = task->file_pos;
|
||||
read_size = task->read_size;
|
||||
|
||||
/* by default no rehash in case of "continue" */
|
||||
rehandle[diskcur].block = 0;
|
||||
ds = &disk_state[diskcur];
|
||||
ds->task = task;
|
||||
ds->diskcur = diskcur;
|
||||
ds->file_is_unsynced = 0;
|
||||
|
||||
/* if the disk position is not used */
|
||||
if (!disk)
|
||||
|
|
@ -407,7 +411,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
if (block_has_invalid_parity(block)) {
|
||||
/* report that the block and the file are not synced */
|
||||
block_is_unsynced = 1;
|
||||
file_is_unsynced = 1;
|
||||
ds->file_is_unsynced = 1;
|
||||
/* follow */
|
||||
}
|
||||
|
||||
|
|
@ -419,7 +423,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
if (task->is_timestamp_different) {
|
||||
/* report that the block and the file are not synced */
|
||||
block_is_unsynced = 1;
|
||||
file_is_unsynced = 1;
|
||||
ds->file_is_unsynced = 1;
|
||||
/* follow */
|
||||
}
|
||||
|
||||
|
|
@ -462,21 +466,43 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
countsize += read_size;
|
||||
countsize += task->read_size;
|
||||
|
||||
/* now compute the hash */
|
||||
if (rehash) {
|
||||
memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
|
||||
/* request hash computation */
|
||||
ds->needs_hash = 1;
|
||||
hash_digest[hash_count] = ds->hash;
|
||||
rehash_digest[hash_count] = ds->rehash;
|
||||
hash_src[hash_count] = buffer[diskcur];
|
||||
hash_size[hash_count] = task->read_size;
|
||||
++hash_count;
|
||||
}
|
||||
|
||||
/* compute the new hash, and store it */
|
||||
rehandle[diskcur].block = block;
|
||||
memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
|
||||
/* for each disk, compute the hash */
|
||||
if (hash_count > 0) {
|
||||
if (needs_rehash) {
|
||||
memhash_multi(state->prevhash, state->prevhashseed, hash_digest, hash_src, hash_size, hash_count);
|
||||
memhash_multi(state->hash, state->hashseed, rehash_digest, hash_src, hash_size, hash_count);
|
||||
} else {
|
||||
memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
|
||||
memhash_multi(state->hash, state->hashseed, hash_digest, hash_src, hash_size, hash_count);
|
||||
}
|
||||
}
|
||||
|
||||
/* until now is hash */
|
||||
state_usage_hash(state);
|
||||
/* until now is hash */
|
||||
state_usage_hash(state);
|
||||
|
||||
/* for each disk, process the hash results */
|
||||
for (j = 0; j < diskmax; ++j) {
|
||||
if (!disk_state[j].needs_hash)
|
||||
continue;
|
||||
|
||||
struct snapraid_task* task = disk_state[j].task;
|
||||
struct snapraid_block* block = task->block;
|
||||
struct snapraid_disk* disk = task->disk;
|
||||
struct snapraid_file* file = task->file;
|
||||
block_off_t file_pos = task->file_pos;
|
||||
|
||||
unsigned char* hash = disk_state[j].hash;
|
||||
int file_is_unsynced = disk_state[j].file_is_unsynced;
|
||||
|
||||
if (block_has_updated_hash(block)) {
|
||||
/* compare the hash */
|
||||
|
|
@ -604,11 +630,11 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
|
|||
*/
|
||||
} else {
|
||||
/* if rehash is needed */
|
||||
if (rehash) {
|
||||
if (needs_rehash) {
|
||||
/* store all the new hash already computed */
|
||||
for (j = 0; j < diskmax; ++j) {
|
||||
if (rehandle[j].block)
|
||||
memcpy(rehandle[j].block->hash, rehandle[j].hash, BLOCK_HASH_SIZE);
|
||||
if (disk_state[j].needs_hash)
|
||||
memcpy(disk_state[j].task->block->hash, disk_state[j].rehash, BLOCK_HASH_SIZE);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -738,7 +764,11 @@ bail:
|
|||
}
|
||||
|
||||
free(handle);
|
||||
free(rehandle_alloc);
|
||||
free(disk_state);
|
||||
free(hash_digest);
|
||||
free(rehash_digest);
|
||||
free(hash_src);
|
||||
free(hash_size);
|
||||
free(waiting_map);
|
||||
io_done(&io);
|
||||
free(block_enabled);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue