mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
76998613da
The same addressing rules are applied to the parity disks because they are used as data disks during recovery operations.
1143 lines
29 KiB
C
1143 lines
29 KiB
C
// SPDX-License-Identifier: GPL-3.0-or-later
|
|
// Copyright (C) 2016 Andrea Mazzoleni
|
|
|
|
#include "portable.h"
|
|
|
|
#include "io.h"
|
|
|
|
void (*io_start)(struct snapraid_io* io,
|
|
block_off_t blockstart, block_off_t blockmax,
|
|
bit_vect_t* block_enabled) = 0;
|
|
void (*io_stop)(struct snapraid_io* io) = 0;
|
|
block_off_t (*io_read_next)(struct snapraid_io* io, void*** buffer) = 0;
|
|
struct snapraid_task* (*io_data_read)(struct snapraid_io* io, unsigned* diskcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
|
|
struct snapraid_task* (*io_parity_read)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
|
|
void (*io_parity_write)(struct snapraid_io* io, unsigned* levcur, unsigned* waiting_map, unsigned* waiting_mac) = 0;
|
|
void (*io_write_preset)(struct snapraid_io* io, block_off_t blockcur, int skip) = 0;
|
|
void (*io_write_next)(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error) = 0;
|
|
void (*io_refresh)(struct snapraid_io* io) = 0;
|
|
void (*io_flush)(struct snapraid_io* io) = 0;
|
|
|
|
/**
|
|
* Get the next block position to process.
|
|
*/
|
|
static block_off_t io_position_next(struct snapraid_io* io)
|
|
{
|
|
block_off_t blockcur;
|
|
|
|
/* get the next position */
|
|
if (io->block_enabled) {
|
|
while (io->block_next < io->block_max && !bit_vect_test(io->block_enabled, io->block_next))
|
|
++io->block_next;
|
|
}
|
|
|
|
blockcur = io->block_next;
|
|
|
|
/* next block for the next call */
|
|
++io->block_next;
|
|
|
|
return blockcur;
|
|
}
|
|
|
|
/**
|
|
* Setup the next pending task for all readers.
|
|
*/
|
|
static void io_reader_sched(struct snapraid_io* io, int task_index, block_off_t blockcur)
|
|
{
|
|
unsigned i;
|
|
|
|
for (i = 0; i < io->reader_max; ++i) {
|
|
struct snapraid_worker* worker = &io->reader_map[i];
|
|
struct snapraid_task* task = &worker->task_map[task_index];
|
|
|
|
/* setup the new pending task */
|
|
if (blockcur < io->block_max)
|
|
task->state = TASK_STATE_READY;
|
|
else
|
|
task->state = TASK_STATE_EMPTY;
|
|
|
|
task->path[0] = 0;
|
|
if (worker->handle)
|
|
task->disk = worker->handle->disk;
|
|
else
|
|
task->disk = 0;
|
|
|
|
assert(worker->buffer_skew + i < io->buffer_max);
|
|
|
|
task->buffer = io->buffer_map[task_index][worker->buffer_skew + i];
|
|
task->position = blockcur;
|
|
task->block = 0;
|
|
task->file = 0;
|
|
task->file_pos = 0;
|
|
task->read_size = 0;
|
|
task->is_timestamp_different = 0;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Setup the next pending task for all writers.
|
|
*/
|
|
static void io_writer_sched(struct snapraid_io* io, int task_index, block_off_t blockcur)
|
|
{
|
|
unsigned i;
|
|
|
|
for (i = 0; i < io->writer_max; ++i) {
|
|
struct snapraid_worker* worker = &io->writer_map[i];
|
|
struct snapraid_task* task = &worker->task_map[task_index];
|
|
|
|
/* setup the new pending task */
|
|
task->state = TASK_STATE_READY;
|
|
task->path[0] = 0;
|
|
task->disk = 0;
|
|
task->buffer = io->buffer_map[task_index][worker->buffer_skew + i];
|
|
task->position = blockcur;
|
|
task->block = 0;
|
|
task->file = 0;
|
|
task->file_pos = 0;
|
|
task->read_size = 0;
|
|
task->is_timestamp_different = 0;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Setup an empty next pending task for all writers.
|
|
*/
|
|
static void io_writer_sched_empty(struct snapraid_io* io, int task_index, block_off_t blockcur)
|
|
{
|
|
unsigned i;
|
|
|
|
for (i = 0; i < io->writer_max; ++i) {
|
|
struct snapraid_worker* worker = &io->writer_map[i];
|
|
struct snapraid_task* task = &worker->task_map[task_index];
|
|
|
|
/* setup the new pending task */
|
|
task->state = TASK_STATE_EMPTY;
|
|
task->path[0] = 0;
|
|
task->disk = 0;
|
|
task->buffer = 0;
|
|
task->position = blockcur;
|
|
task->block = 0;
|
|
task->file = 0;
|
|
task->file_pos = 0;
|
|
task->read_size = 0;
|
|
task->is_timestamp_different = 0;
|
|
}
|
|
}
|
|
|
|
/*****************************************************************************/
|
|
/* mono thread */
|
|
|
|
static block_off_t io_read_next_mono(struct snapraid_io* io, void*** buffer)
|
|
{
|
|
block_off_t blockcur_schedule;
|
|
|
|
/* reset the index */
|
|
io->reader_index = 0;
|
|
|
|
blockcur_schedule = io_position_next(io);
|
|
|
|
/* schedule the next read */
|
|
io_reader_sched(io, 0, blockcur_schedule);
|
|
|
|
/* set the buffer to use */
|
|
*buffer = io->buffer_map[0];
|
|
|
|
return blockcur_schedule;
|
|
}
|
|
|
|
static void io_write_preset_mono(struct snapraid_io* io, block_off_t blockcur, int skip)
|
|
{
|
|
unsigned i;
|
|
|
|
/* reset the index */
|
|
io->writer_index = 0;
|
|
|
|
/* clear errors */
|
|
for (i = 0; i < IO_WRITER_ERROR_MAX; ++i)
|
|
io->writer_error[i] = 0;
|
|
|
|
if (skip) {
|
|
/* skip the next write */
|
|
io_writer_sched_empty(io, 0, blockcur);
|
|
} else {
|
|
/* schedule the next write */
|
|
io_writer_sched(io, 0, blockcur);
|
|
}
|
|
}
|
|
|
|
static void io_write_next_mono(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error)
|
|
{
|
|
unsigned i;
|
|
|
|
(void)blockcur;
|
|
(void)skip;
|
|
|
|
/* report errors */
|
|
for (i = 0; i < IO_WRITER_ERROR_MAX; ++i)
|
|
writer_error[i] = io->writer_error[i];
|
|
}
|
|
|
|
static void io_refresh_mono(struct snapraid_io* io)
|
|
{
|
|
(void)io;
|
|
}
|
|
|
|
static void io_flush_mono(struct snapraid_io* io)
|
|
{
|
|
(void)io;
|
|
}
|
|
|
|
static struct snapraid_task* io_task_read_mono(struct snapraid_io* io, unsigned base, unsigned count, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
struct snapraid_worker* worker;
|
|
struct snapraid_task* task;
|
|
unsigned i;
|
|
|
|
/* get the next task */
|
|
i = io->reader_index++;
|
|
|
|
assert(base <= i && i < base + count);
|
|
|
|
worker = &io->reader_map[i];
|
|
task = &worker->task_map[0];
|
|
|
|
/* do the work */
|
|
if (task->state != TASK_STATE_EMPTY)
|
|
worker->func(worker, task);
|
|
|
|
/* return the position */
|
|
*pos = i - base;
|
|
|
|
/* store the waiting index */
|
|
waiting_map[0] = i - base;
|
|
*waiting_mac = 1;
|
|
|
|
return task;
|
|
}
|
|
|
|
static struct snapraid_task* io_data_read_mono(struct snapraid_io* io, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
return io_task_read_mono(io, io->data_base, io->data_count, pos, waiting_map, waiting_mac);
|
|
}
|
|
|
|
static struct snapraid_task* io_parity_read_mono(struct snapraid_io* io, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
return io_task_read_mono(io, io->parity_base, io->parity_count, pos, waiting_map, waiting_mac);
|
|
}
|
|
|
|
static void io_parity_write_mono(struct snapraid_io* io, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
struct snapraid_worker* worker;
|
|
struct snapraid_task* task;
|
|
unsigned i;
|
|
|
|
/* get the next task */
|
|
i = io->writer_index++;
|
|
|
|
worker = &io->writer_map[i];
|
|
task = &worker->task_map[0];
|
|
|
|
io->writer_error[i] = 0;
|
|
|
|
/* do the work */
|
|
if (task->state != TASK_STATE_EMPTY)
|
|
worker->func(worker, task);
|
|
|
|
/* return the position */
|
|
*pos = i;
|
|
|
|
/* store the waiting index */
|
|
waiting_map[0] = i;
|
|
*waiting_mac = 1;
|
|
}
|
|
|
|
static void io_start_mono(struct snapraid_io* io,
|
|
block_off_t blockstart, block_off_t blockmax,
|
|
bit_vect_t* block_enabled)
|
|
{
|
|
io->block_start = blockstart;
|
|
io->block_max = blockmax;
|
|
io->block_enabled = block_enabled;
|
|
io->block_next = blockstart;
|
|
}
|
|
|
|
static void io_stop_mono(struct snapraid_io* io)
|
|
{
|
|
(void)io;
|
|
}
|
|
|
|
/*****************************************************************************/
|
|
/* multi thread */
|
|
|
|
/* disable multithread if pthread is not present */
|
|
#if HAVE_THREAD
|
|
|
|
/**
|
|
* Get the next task to work on for a reader.
|
|
*
|
|
* This is the synchronization point for workers with the io.
|
|
*/
|
|
static struct snapraid_task* io_reader_step(struct snapraid_worker* worker)
|
|
{
|
|
struct snapraid_io* io = worker->io;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
while (1) {
|
|
unsigned next_index;
|
|
|
|
/*
|
|
* Check if the worker has to exit
|
|
* even if there is work to do
|
|
*/
|
|
if (io->done) {
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
return 0;
|
|
}
|
|
|
|
/* get the next pending task */
|
|
next_index = (worker->index + 1) % io->io_max;
|
|
|
|
/* if the queue of pending tasks is not empty */
|
|
if (next_index != io->reader_index) {
|
|
struct snapraid_task* task;
|
|
|
|
/* the index that the IO may be waiting for */
|
|
unsigned waiting_index = io->reader_index;
|
|
|
|
/* the index that worker just completed */
|
|
unsigned done_index = worker->index;
|
|
|
|
/* get the new working task */
|
|
worker->index = next_index;
|
|
task = &worker->task_map[worker->index];
|
|
|
|
/* if the just completed task is at this index */
|
|
if (done_index == waiting_index) {
|
|
/* notify the IO that a new read is complete */
|
|
thread_cond_signal_and_unlock(&io->read_done, &io->io_mutex);
|
|
} else {
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
}
|
|
|
|
/* return the new task */
|
|
return task;
|
|
}
|
|
|
|
/* otherwise wait for a read_sched event */
|
|
thread_cond_wait(&io->read_sched, &io->io_mutex);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the next task to work on for a writer.
|
|
*
|
|
* This is the synchronization point for workers with the io.
|
|
*/
|
|
static struct snapraid_task* io_writer_step(struct snapraid_worker* worker, int state)
|
|
{
|
|
struct snapraid_io* io = worker->io;
|
|
int error_index;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
/* counts the number of errors in the global state */
|
|
error_index = state - IO_WRITER_ERROR_BASE;
|
|
if (error_index >= 0 && error_index < IO_WRITER_ERROR_MAX)
|
|
++io->writer_error[error_index];
|
|
|
|
while (1) {
|
|
unsigned next_index;
|
|
|
|
/* get the next pending task */
|
|
next_index = (worker->index + 1) % io->io_max;
|
|
|
|
/* if the queue of pending tasks is not empty */
|
|
if (next_index != io->writer_index) {
|
|
struct snapraid_task* task;
|
|
|
|
/* the index that the IO may be waiting for */
|
|
unsigned waiting_index = (io->writer_index + 1) % io->io_max;
|
|
|
|
/* the index that worker just completed */
|
|
unsigned done_index = worker->index;
|
|
|
|
/* get the new working task */
|
|
worker->index = next_index;
|
|
task = &worker->task_map[worker->index];
|
|
|
|
/* if the just completed task is at this index */
|
|
if (done_index == waiting_index) {
|
|
/* notify the IO that a new write is complete */
|
|
thread_cond_signal_and_unlock(&io->write_done, &io->io_mutex);
|
|
} else {
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
}
|
|
|
|
/* return the new task */
|
|
return task;
|
|
}
|
|
|
|
/*
|
|
* Check if the worker has to exit
|
|
* but only if there is no work to do
|
|
*/
|
|
if (io->done) {
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
return 0;
|
|
}
|
|
|
|
/* otherwise wait for a write_sched event */
|
|
thread_cond_wait(&io->write_sched, &io->io_mutex);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Get the next block position to operate on.
|
|
*
|
|
* This is the synchronization point for workers with the io.
|
|
*/
|
|
static block_off_t io_read_next_thread(struct snapraid_io* io, void*** buffer)
|
|
{
|
|
block_off_t blockcur_schedule;
|
|
block_off_t blockcur_caller;
|
|
unsigned i;
|
|
|
|
/* get the next parity position to process */
|
|
blockcur_schedule = io_position_next(io);
|
|
|
|
/* ensure that all data/parity was read */
|
|
assert(io->reader_list[0] == io->reader_max);
|
|
|
|
/* setup the list of workers to process */
|
|
for (i = 0; i <= io->reader_max; ++i)
|
|
io->reader_list[i] = i;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
/* schedule the next read */
|
|
io_reader_sched(io, io->reader_index, blockcur_schedule);
|
|
|
|
/* set the index for the tasks to return to the caller */
|
|
io->reader_index = (io->reader_index + 1) % io->io_max;
|
|
|
|
/* get the position to operate at high level from one task */
|
|
blockcur_caller = io->reader_map[0].task_map[io->reader_index].position;
|
|
|
|
/* set the buffer to use */
|
|
*buffer = io->buffer_map[io->reader_index];
|
|
|
|
/* signal all the workers that there is a new pending task */
|
|
thread_cond_broadcast_and_unlock(&io->read_sched, &io->io_mutex);
|
|
|
|
return blockcur_caller;
|
|
}
|
|
|
|
static void io_write_preset_thread(struct snapraid_io* io, block_off_t blockcur, int skip)
|
|
{
|
|
(void)io;
|
|
(void)blockcur;
|
|
(void)skip;
|
|
}
|
|
|
|
static void io_write_next_thread(struct snapraid_io* io, block_off_t blockcur, int skip, int* writer_error)
|
|
{
|
|
unsigned i;
|
|
|
|
/* ensure that all parity was written */
|
|
assert(io->writer_list[0] == io->writer_max);
|
|
|
|
/* setup the list of workers to process */
|
|
for (i = 0; i <= io->writer_max; ++i)
|
|
io->writer_list[i] = i;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
/* report errors */
|
|
for (i = 0; i < IO_WRITER_ERROR_MAX; ++i) {
|
|
writer_error[i] = io->writer_error[i];
|
|
io->writer_error[i] = 0;
|
|
}
|
|
|
|
if (skip) {
|
|
/* skip the next write */
|
|
io_writer_sched_empty(io, io->writer_index, blockcur);
|
|
} else {
|
|
/* schedule the next write */
|
|
io_writer_sched(io, io->writer_index, blockcur);
|
|
}
|
|
|
|
/* at this point the writers must be in sync with the readers */
|
|
assert(io->writer_index == io->reader_index);
|
|
|
|
/* set the index to be used for the next write */
|
|
io->writer_index = (io->writer_index + 1) % io->io_max;
|
|
|
|
/* signal all the workers that there is a new pending task */
|
|
thread_cond_broadcast_and_unlock(&io->write_sched, &io->io_mutex);
|
|
}
|
|
|
|
static void io_refresh_thread(struct snapraid_io* io)
|
|
{
|
|
unsigned i;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
/* for all readers, count the number of read blocks */
|
|
for (i = 0; i < io->reader_max; ++i) {
|
|
unsigned begin, end, cached;
|
|
struct snapraid_worker* worker = &io->reader_map[i];
|
|
|
|
/* the first block read */
|
|
begin = io->reader_index + 1;
|
|
/* the block in reading */
|
|
end = worker->index;
|
|
if (begin > end)
|
|
end += io->io_max;
|
|
cached = end - begin;
|
|
|
|
if (worker->parity_handle)
|
|
io->state->parity[worker->parity_handle->level].cached_blocks = cached;
|
|
else
|
|
worker->handle->disk->cached_blocks = cached;
|
|
}
|
|
|
|
/*
|
|
* For all writers, count the number of written blocks
|
|
* note that this is a kind of "opposite" of cached blocks
|
|
*/
|
|
for (i = 0; i < io->writer_max; ++i) {
|
|
unsigned begin, end, cached;
|
|
struct snapraid_worker* worker = &io->writer_map[i];
|
|
|
|
/* the first block written */
|
|
begin = io->writer_index + 1;
|
|
/* the block in writing */
|
|
end = worker->index;
|
|
if (begin > end)
|
|
end += io->io_max;
|
|
cached = end - begin;
|
|
|
|
io->state->parity[worker->parity_handle->level].cached_blocks = cached;
|
|
}
|
|
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
}
|
|
|
|
static void io_flush_thread(struct snapraid_io* io)
|
|
{
|
|
unsigned i;
|
|
|
|
while (1) {
|
|
int all_done = 1;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
for (i = 0; i < io->writer_max; ++i) {
|
|
struct snapraid_worker* worker = &io->writer_map[i];
|
|
|
|
/* get the next pending task */
|
|
unsigned next_index = (worker->index + 1) % io->io_max;
|
|
|
|
/* if the queue of pending tasks is not empty */
|
|
if (next_index != io->writer_index) {
|
|
all_done = 0;
|
|
break;
|
|
}
|
|
}
|
|
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
|
|
if (all_done)
|
|
break;
|
|
|
|
/* wait for something to complete */
|
|
thread_yield();
|
|
}
|
|
}
|
|
|
|
static struct snapraid_task* io_task_read_thread(struct snapraid_io* io, unsigned base, unsigned count, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
unsigned waiting_cycle;
|
|
|
|
/* count the waiting cycle */
|
|
waiting_cycle = 0;
|
|
|
|
/* clear the waiting indexes */
|
|
*waiting_mac = 0;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
while (1) {
|
|
unsigned* let;
|
|
unsigned busy_index;
|
|
|
|
/*
|
|
* Get the index the IO is using
|
|
* we must ensure that this index has not a read in progress
|
|
* to avoid a concurrent access
|
|
*/
|
|
busy_index = io->reader_index;
|
|
|
|
/* search for a worker that has already finished */
|
|
let = &io->reader_list[0];
|
|
while (1) {
|
|
unsigned i = *let;
|
|
|
|
/* if we are at the end */
|
|
if (i == io->reader_max)
|
|
break;
|
|
|
|
/* if it's in range */
|
|
if (base <= i && i < base + count) {
|
|
struct snapraid_worker* worker;
|
|
|
|
/* if it's the first cycle */
|
|
if (waiting_cycle == 0) {
|
|
/* store the waiting indexes */
|
|
waiting_map[(*waiting_mac)++] = i - base;
|
|
}
|
|
|
|
worker = &io->reader_map[i];
|
|
|
|
/* if the worker has finished this index */
|
|
if (busy_index != worker->index) {
|
|
struct snapraid_task* task;
|
|
|
|
task = &worker->task_map[io->reader_index];
|
|
|
|
/*
|
|
* Mark the worker as processed
|
|
* setting the previous one to point at the next one
|
|
*/
|
|
*let = io->reader_list[i + 1];
|
|
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
|
|
/* return the position */
|
|
*pos = i - base;
|
|
|
|
/* on the first cycle, no one is waiting */
|
|
if (waiting_cycle == 0)
|
|
*waiting_mac = 0;
|
|
|
|
return task;
|
|
}
|
|
}
|
|
|
|
/* next position to check */
|
|
let = &io->reader_list[i + 1];
|
|
}
|
|
|
|
/* if no worker is ready, wait for an event */
|
|
thread_cond_wait(&io->read_done, &io->io_mutex);
|
|
|
|
/* count the cycles */
|
|
++waiting_cycle;
|
|
}
|
|
}
|
|
|
|
static struct snapraid_task* io_data_read_thread(struct snapraid_io* io, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
return io_task_read_thread(io, io->data_base, io->data_count, pos, waiting_map, waiting_mac);
|
|
}
|
|
|
|
static struct snapraid_task* io_parity_read_thread(struct snapraid_io* io, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
return io_task_read_thread(io, io->parity_base, io->parity_count, pos, waiting_map, waiting_mac);
|
|
}
|
|
|
|
static void io_parity_write_thread(struct snapraid_io* io, unsigned* pos, unsigned* waiting_map, unsigned* waiting_mac)
|
|
{
|
|
unsigned waiting_cycle;
|
|
|
|
/* count the waiting cycle */
|
|
waiting_cycle = 0;
|
|
|
|
/* clear the waiting indexes */
|
|
*waiting_mac = 0;
|
|
|
|
/* the synchronization is protected by the io mutex */
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
while (1) {
|
|
unsigned* let;
|
|
unsigned busy_index;
|
|
|
|
/*
|
|
* Get the next index the IO is going to use
|
|
* we must ensure that this index has not a write in progress
|
|
* to avoid a concurrent access
|
|
* note that we are already sure that a write is not in progress
|
|
* at the index the IO is using at now
|
|
*/
|
|
busy_index = (io->writer_index + 1) % io->io_max;
|
|
|
|
/* search for a worker that has already finished */
|
|
let = &io->writer_list[0];
|
|
while (1) {
|
|
unsigned i = *let;
|
|
struct snapraid_worker* worker;
|
|
|
|
/* if we are at the end */
|
|
if (i == io->writer_max)
|
|
break;
|
|
|
|
/* if it's the first cycle */
|
|
if (waiting_cycle == 0) {
|
|
/* store the waiting indexes */
|
|
waiting_map[(*waiting_mac)++] = i;
|
|
}
|
|
|
|
worker = &io->writer_map[i];
|
|
|
|
/* the two indexes cannot be equal */
|
|
assert(io->writer_index != worker->index);
|
|
|
|
/* if the worker has finished this index */
|
|
if (busy_index != worker->index) {
|
|
/*
|
|
* Mark the worker as processed
|
|
* setting the previous one to point at the next one
|
|
*/
|
|
*let = io->writer_list[i + 1];
|
|
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
|
|
/* return the position */
|
|
*pos = i;
|
|
|
|
/* on the first cycle, no one is waiting */
|
|
if (waiting_cycle == 0)
|
|
*waiting_mac = 0;
|
|
|
|
return;
|
|
}
|
|
|
|
/* next position to check */
|
|
let = &io->writer_list[i + 1];
|
|
}
|
|
|
|
/* if no worker is ready, wait for an event */
|
|
thread_cond_wait(&io->write_done, &io->io_mutex);
|
|
|
|
/* count the cycles */
|
|
++waiting_cycle;
|
|
}
|
|
}
|
|
|
|
static void io_reader_worker(struct snapraid_worker* worker, struct snapraid_task* task)
|
|
{
|
|
/* if we reached the end */
|
|
if (task->position >= worker->io->block_max) {
|
|
/* complete a dummy task */
|
|
task->state = TASK_STATE_EMPTY;
|
|
} else {
|
|
worker->func(worker, task);
|
|
}
|
|
}
|
|
|
|
static void* io_reader_thread(void* arg)
|
|
{
|
|
struct snapraid_worker* worker = arg;
|
|
|
|
/* force completion of the first task */
|
|
io_reader_worker(worker, &worker->task_map[0]);
|
|
|
|
while (1) {
|
|
struct snapraid_task* task;
|
|
|
|
/* get the new task */
|
|
task = io_reader_step(worker);
|
|
|
|
/* if no task, it means to exit */
|
|
if (!task)
|
|
break;
|
|
|
|
/* nothing more to do */
|
|
if (task->state == TASK_STATE_EMPTY)
|
|
continue;
|
|
|
|
if (task->state != TASK_STATE_READY) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency: Unexpected state %d in reader for task at position %u in worker %u\n", task->state, task->position, worker->index);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* work on the assigned task */
|
|
io_reader_worker(worker, task);
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void* io_writer_thread(void* arg)
|
|
{
|
|
struct snapraid_worker* worker = arg;
|
|
int latest_state = TASK_STATE_DONE;
|
|
|
|
while (1) {
|
|
struct snapraid_task* task;
|
|
|
|
/* get the new task */
|
|
task = io_writer_step(worker, latest_state);
|
|
|
|
/* if no task, it means to exit */
|
|
if (!task)
|
|
break;
|
|
|
|
/* nothing more to do */
|
|
if (task->state == TASK_STATE_EMPTY) {
|
|
latest_state = TASK_STATE_DONE;
|
|
continue;
|
|
}
|
|
|
|
if (task->state != TASK_STATE_READY) {
|
|
/* LCOV_EXCL_START */
|
|
log_fatal(EINTERNAL, "Internal inconsistency: Unexpected state %d in writer for task at position %u in worker %u\n", task->state, task->position, worker->index);
|
|
os_abort();
|
|
/* LCOV_EXCL_STOP */
|
|
}
|
|
|
|
/* work on the assigned task */
|
|
worker->func(worker, task);
|
|
|
|
/* save the resulting state */
|
|
latest_state = task->state;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void io_start_thread(struct snapraid_io* io,
|
|
block_off_t blockstart, block_off_t blockmax,
|
|
bit_vect_t* block_enabled)
|
|
{
|
|
unsigned i;
|
|
|
|
io->block_start = blockstart;
|
|
io->block_max = blockmax;
|
|
io->block_enabled = block_enabled;
|
|
io->block_next = blockstart;
|
|
|
|
io->done = 0;
|
|
io->reader_index = io->io_max - 1; /* the first io_read_next() is going to set it to 0 */
|
|
io->writer_index = 0;
|
|
|
|
/* clear writer errors */
|
|
for (i = 0; i < IO_WRITER_ERROR_MAX; ++i)
|
|
io->writer_error[i] = 0;
|
|
|
|
/*
|
|
* Setup the initial read pending tasks, except the latest one,
|
|
* the latest will be initialized at the fist io_read_next() call
|
|
*/
|
|
for (i = 0; i < io->io_max - 1; ++i) {
|
|
block_off_t blockcur = io_position_next(io);
|
|
|
|
io_reader_sched(io, i, blockcur);
|
|
}
|
|
|
|
/* setup the lists of workers to process */
|
|
io->reader_list[0] = io->reader_max;
|
|
for (i = 0; i <= io->writer_max; ++i)
|
|
io->writer_list[i] = i;
|
|
|
|
/* start the reader threads */
|
|
for (i = 0; i < io->reader_max; ++i) {
|
|
struct snapraid_worker* worker = &io->reader_map[i];
|
|
|
|
worker->index = 0;
|
|
|
|
thread_create(&worker->thread, io_reader_thread, worker);
|
|
}
|
|
|
|
/* start the writer threads */
|
|
for (i = 0; i < io->writer_max; ++i) {
|
|
struct snapraid_worker* worker = &io->writer_map[i];
|
|
|
|
worker->index = io->io_max - 1;
|
|
|
|
thread_create(&worker->thread, io_writer_thread, worker);
|
|
}
|
|
}
|
|
|
|
static void io_stop_thread(struct snapraid_io* io)
|
|
{
|
|
unsigned i;
|
|
|
|
thread_mutex_lock(&io->io_mutex);
|
|
|
|
/* mark that we are stopping */
|
|
io->done = 1;
|
|
|
|
/* signal all the threads to recognize the new state */
|
|
thread_cond_broadcast(&io->read_sched);
|
|
thread_cond_broadcast(&io->write_sched);
|
|
|
|
thread_mutex_unlock(&io->io_mutex);
|
|
|
|
/* wait for all readers to terminate */
|
|
for (i = 0; i < io->reader_max; ++i) {
|
|
struct snapraid_worker* worker = &io->reader_map[i];
|
|
void* retval;
|
|
|
|
/* wait for thread termination */
|
|
thread_join(worker->thread, &retval);
|
|
}
|
|
|
|
/* wait for all writers to terminate */
|
|
for (i = 0; i < io->writer_max; ++i) {
|
|
struct snapraid_worker* worker = &io->writer_map[i];
|
|
void* retval;
|
|
|
|
/* wait for thread termination */
|
|
thread_join(worker->thread, &retval);
|
|
}
|
|
}
|
|
|
|
#endif
|
|
|
|
/*****************************************************************************/
|
|
/* global */
|
|
|
|
void io_init(struct snapraid_io* io, struct snapraid_state* state,
|
|
unsigned io_cache, unsigned buffer_max,
|
|
struct snapraid_handle* handle_map, unsigned handle_max,
|
|
const io_op_t* data_ops,
|
|
void (*data_reader)(struct snapraid_worker*, struct snapraid_task*),
|
|
void (*data_writer)(struct snapraid_worker*, struct snapraid_task*),
|
|
struct snapraid_parity_handle* parity_handle_map, unsigned parity_handle_max,
|
|
const io_op_t* parity_ops,
|
|
void (*parity_reader)(struct snapraid_worker*, struct snapraid_task*),
|
|
void (*parity_writer)(struct snapraid_worker*, struct snapraid_task*))
|
|
{
|
|
unsigned i;
|
|
size_t allocated_size;
|
|
size_t block_size = state->block_size;
|
|
unsigned readers_count = 0;
|
|
unsigned writers_count = 0;
|
|
unsigned r_idx = 0;
|
|
unsigned w_idx = 0;
|
|
io_op_t data_op_default = IO_OP_NONE;
|
|
io_op_t parity_op_default = IO_OP_NONE;
|
|
|
|
io->state = state;
|
|
|
|
assert(buffer_max >= handle_max + parity_handle_max);
|
|
|
|
/* initialize bandwidth limiting */
|
|
bw_init(&io->bw, state->opt.bwlimit);
|
|
|
|
/* set IO context in handles */
|
|
for (i = 0; i < handle_max; ++i)
|
|
handle_map[i].bw = &io->bw;
|
|
for (i = 0; i < parity_handle_max; ++i)
|
|
parity_handle_map[i].bw = &io->bw;
|
|
|
|
#if HAVE_THREAD
|
|
if (io_cache == 0) {
|
|
/*
|
|
* Default is 16 MiB of cache
|
|
* this seems to be a good tradeoff between speed and memory usage
|
|
*/
|
|
io->io_max = 16 * 1024 * 1024 / state->block_size;
|
|
if (io->io_max < IO_MIN)
|
|
io->io_max = IO_MIN;
|
|
if (io->io_max > IO_MAX)
|
|
io->io_max = IO_MAX;
|
|
} else {
|
|
io->io_max = io_cache;
|
|
}
|
|
#else
|
|
(void)io_cache;
|
|
|
|
/* without pthread force the mono thread mode */
|
|
io->io_max = 1;
|
|
#endif
|
|
|
|
assert(io->io_max == 1 || (io->io_max >= IO_MIN && io->io_max <= IO_MAX));
|
|
|
|
io->buffer_max = buffer_max;
|
|
allocated_size = 0;
|
|
for (i = 0; i < io->io_max; ++i) {
|
|
if (state->file_mode != ADVISE_DIRECT)
|
|
io->buffer_map[i] = malloc_nofail_vector_align(buffer_max, block_size, &io->buffer_alloc_map[i]);
|
|
else
|
|
io->buffer_map[i] = malloc_nofail_vector_direct(buffer_max, block_size, &io->buffer_alloc_map[i]);
|
|
if (!state->opt.skip_self)
|
|
mtest_vector(io->buffer_max, state->block_size, io->buffer_map[i]);
|
|
allocated_size += block_size * buffer_max;
|
|
}
|
|
|
|
msg_progress("Using %u MiB of memory for %u cached blocks.\n", (unsigned)(allocated_size / MEBI), io->io_max);
|
|
|
|
/* resolve implicit fallback directions if ops are NULL */
|
|
if (data_reader && !data_writer) {
|
|
data_op_default = IO_OP_READ;
|
|
}
|
|
if (!data_reader && data_writer) {
|
|
data_op_default = IO_OP_WRITE;
|
|
}
|
|
if (parity_reader && !parity_writer) {
|
|
parity_op_default = IO_OP_READ;
|
|
}
|
|
if (!parity_reader && parity_writer) {
|
|
parity_op_default = IO_OP_WRITE;
|
|
}
|
|
|
|
/* count reader and writer workers */
|
|
for (i = 0; i < handle_max; ++i) {
|
|
io_op_t op = data_ops ? data_ops[i] : data_op_default;
|
|
if (op == IO_OP_READ)
|
|
readers_count++;
|
|
if (op == IO_OP_WRITE)
|
|
writers_count++;
|
|
}
|
|
for (i = 0; i < parity_handle_max; ++i) {
|
|
io_op_t op = parity_ops ? parity_ops[i] : parity_op_default;
|
|
if (op == IO_OP_READ)
|
|
readers_count++;
|
|
if (op == IO_OP_WRITE)
|
|
writers_count++;
|
|
}
|
|
|
|
io->reader_max = readers_count;
|
|
io->writer_max = writers_count;
|
|
|
|
io->reader_map = malloc_nofail(sizeof(struct snapraid_worker) * io->reader_max);
|
|
io->reader_list = malloc_nofail(sizeof(unsigned) * (io->reader_max + 1));
|
|
io->writer_map = malloc_nofail(sizeof(struct snapraid_worker) * io->writer_max);
|
|
io->writer_list = malloc_nofail(sizeof(unsigned) * (io->writer_max + 1));
|
|
|
|
io->data_base = 0;
|
|
io->data_count = 0;
|
|
for (i = 0; i < handle_max; ++i) {
|
|
io_op_t op = data_ops ? data_ops[i] : data_op_default;
|
|
if (op == IO_OP_READ)
|
|
io->data_count++;
|
|
}
|
|
|
|
io->parity_base = io->data_count;
|
|
io->parity_count = 0;
|
|
for (i = 0; i < parity_handle_max; ++i) {
|
|
io_op_t op = parity_ops ? parity_ops[i] : parity_op_default;
|
|
if (op == IO_OP_READ)
|
|
io->parity_count++;
|
|
}
|
|
|
|
/* configure reader map workers */
|
|
for (i = 0; i < handle_max; ++i) {
|
|
io_op_t op = data_ops ? data_ops[i] : data_op_default;
|
|
if (op == IO_OP_READ) {
|
|
struct snapraid_worker* worker = &io->reader_map[r_idx++];
|
|
worker->io = io;
|
|
worker->handle = &handle_map[i];
|
|
worker->parity_handle = 0;
|
|
worker->func = data_reader;
|
|
worker->buffer_skew = i - (r_idx - 1);
|
|
}
|
|
}
|
|
for (i = 0; i < parity_handle_max; ++i) {
|
|
io_op_t op = parity_ops ? parity_ops[i] : parity_op_default;
|
|
if (op == IO_OP_READ) {
|
|
struct snapraid_worker* worker = &io->reader_map[r_idx++];
|
|
worker->io = io;
|
|
worker->handle = 0;
|
|
worker->parity_handle = &parity_handle_map[i];
|
|
worker->func = parity_reader;
|
|
worker->buffer_skew = (handle_max + parity_handle_max + i) - (r_idx - 1);
|
|
}
|
|
}
|
|
|
|
/* configure writer map workers */
|
|
for (i = 0; i < handle_max; ++i) {
|
|
io_op_t op = data_ops ? data_ops[i] : data_op_default;
|
|
if (op == IO_OP_WRITE) {
|
|
struct snapraid_worker* worker = &io->writer_map[w_idx++];
|
|
worker->io = io;
|
|
worker->handle = &handle_map[i];
|
|
worker->parity_handle = 0;
|
|
worker->func = data_writer;
|
|
worker->buffer_skew = i - (w_idx - 1);
|
|
}
|
|
}
|
|
for (i = 0; i < parity_handle_max; ++i) {
|
|
io_op_t op = parity_ops ? parity_ops[i] : parity_op_default;
|
|
if (op == IO_OP_WRITE) {
|
|
struct snapraid_worker* worker = &io->writer_map[w_idx++];
|
|
worker->io = io;
|
|
worker->handle = 0;
|
|
worker->parity_handle = &parity_handle_map[i];
|
|
worker->func = parity_writer;
|
|
worker->buffer_skew = (handle_max + i) - (w_idx - 1);
|
|
}
|
|
}
|
|
|
|
#if HAVE_THREAD
|
|
if (io->io_max > 1) {
|
|
io_read_next = io_read_next_thread;
|
|
io_write_preset = io_write_preset_thread;
|
|
io_write_next = io_write_next_thread;
|
|
io_refresh = io_refresh_thread;
|
|
io_flush = io_flush_thread;
|
|
io_data_read = io_data_read_thread;
|
|
io_parity_read = io_parity_read_thread;
|
|
io_parity_write = io_parity_write_thread;
|
|
io_start = io_start_thread;
|
|
io_stop = io_stop_thread;
|
|
|
|
thread_mutex_init(&io->io_mutex);
|
|
thread_cond_init(&io->read_done);
|
|
thread_cond_init(&io->read_sched);
|
|
thread_cond_init(&io->write_done);
|
|
thread_cond_init(&io->write_sched);
|
|
} else
|
|
#endif
|
|
{
|
|
io_read_next = io_read_next_mono;
|
|
io_write_preset = io_write_preset_mono;
|
|
io_write_next = io_write_next_mono;
|
|
io_refresh = io_refresh_mono;
|
|
io_flush = io_flush_mono;
|
|
io_data_read = io_data_read_mono;
|
|
io_parity_read = io_parity_read_mono;
|
|
io_parity_write = io_parity_write_mono;
|
|
io_start = io_start_mono;
|
|
io_stop = io_stop_mono;
|
|
}
|
|
}
|
|
|
|
void io_done(struct snapraid_io* io)
|
|
{
|
|
unsigned i;
|
|
|
|
for (i = 0; i < io->io_max; ++i) {
|
|
free(io->buffer_map[i]);
|
|
free(io->buffer_alloc_map[i]);
|
|
}
|
|
|
|
free(io->reader_map);
|
|
free(io->reader_list);
|
|
free(io->writer_map);
|
|
free(io->writer_list);
|
|
|
|
#if HAVE_THREAD
|
|
if (io->io_max > 1) {
|
|
thread_mutex_destroy(&io->io_mutex);
|
|
thread_cond_destroy(&io->read_done);
|
|
thread_cond_destroy(&io->read_sched);
|
|
thread_cond_destroy(&io->write_done);
|
|
thread_cond_destroy(&io->write_sched);
|
|
}
|
|
#endif
|
|
}
|
|
|