mirror of
https://github.com/trapexit/mergerfs.git
synced 2026-10-26 19:01:56 -04:00
Rework thread pool for increased stability + config and doc updates (#1453)
Also updates fmt to 11.2.0 and replaced syslog usage with a wrapper leveraging fmt.
This commit is contained in:
trapexit
GitHub
parent
455a81290b
commit
344014aa81
58 changed files with 9097 additions and 24020 deletions
|
|
@ -3,10 +3,11 @@
|
|||
#endif
|
||||
|
||||
#include "cpu.hpp"
|
||||
#include "pin_threads.hpp"
|
||||
#include "fmt/core.h"
|
||||
#include "make_unique.hpp"
|
||||
#include "scope_guard.hpp"
|
||||
#include "thread_pool.hpp"
|
||||
#include "syslog.hpp"
|
||||
|
||||
#include "fuse_i.h"
|
||||
#include "fuse_kernel.h"
|
||||
|
|
@ -17,6 +18,10 @@
|
|||
#include "fuse_msgbuf.hpp"
|
||||
#include "fuse_ll.hpp"
|
||||
|
||||
#include <cassert>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include <errno.h>
|
||||
#include <pthread.h>
|
||||
#include <semaphore.h>
|
||||
|
|
@ -25,15 +30,12 @@
|
|||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <sys/time.h>
|
||||
#include <syslog.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cassert>
|
||||
#include <vector>
|
||||
|
||||
static
|
||||
bool
|
||||
retriable_receive_error(const int err_)
|
||||
_retriable_receive_error(const int err_)
|
||||
{
|
||||
switch(err_)
|
||||
{
|
||||
|
|
@ -46,20 +48,10 @@ retriable_receive_error(const int err_)
|
|||
}
|
||||
}
|
||||
|
||||
static
|
||||
bool
|
||||
fatal_receive_error(const int err_)
|
||||
{
|
||||
return (err_ < 0);
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
handle_receive_error(const int rv_,
|
||||
fuse_msgbuf_t *msgbuf_)
|
||||
_print_error(int rv_)
|
||||
{
|
||||
msgbuf_free(msgbuf_);
|
||||
|
||||
fmt::print(stderr,
|
||||
"mergerfs: error reading from /dev/fuse - {} ({})\n",
|
||||
strerror(-rv_),
|
||||
|
|
@ -72,8 +64,8 @@ struct AsyncWorker
|
|||
sem_t *_finished;
|
||||
std::shared_ptr<ThreadPool> _process_tp;
|
||||
|
||||
AsyncWorker(fuse_session *se_,
|
||||
sem_t *finished_,
|
||||
AsyncWorker(fuse_session *se_,
|
||||
sem_t *finished_,
|
||||
std::shared_ptr<ThreadPool> process_tp_)
|
||||
: _se(se_),
|
||||
_finished(finished_),
|
||||
|
|
@ -88,7 +80,7 @@ struct AsyncWorker
|
|||
DEFER{ fuse_session_exit(_se); };
|
||||
DEFER{ sem_post(_finished); };
|
||||
|
||||
moodycamel::ProducerToken ptok(_process_tp->ptoken());
|
||||
ThreadPool::PToken ptok(_process_tp->ptoken());
|
||||
while(!fuse_session_exited(_se))
|
||||
{
|
||||
int rv;
|
||||
|
|
@ -96,26 +88,30 @@ struct AsyncWorker
|
|||
|
||||
msgbuf = msgbuf_alloc();
|
||||
|
||||
do
|
||||
while(true)
|
||||
{
|
||||
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE,NULL);
|
||||
rv = _se->receive_buf(_se,msgbuf);
|
||||
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,NULL);
|
||||
if(rv == 0)
|
||||
return;
|
||||
if(retriable_receive_error(rv))
|
||||
|
||||
if(rv > 0)
|
||||
break;
|
||||
if(::_retriable_receive_error(rv))
|
||||
continue;
|
||||
if(fatal_receive_error(rv))
|
||||
return handle_receive_error(rv,msgbuf);
|
||||
} while(false);
|
||||
|
||||
auto const func = [=]
|
||||
{
|
||||
_se->process_buf(_se,msgbuf);
|
||||
msgbuf_free(msgbuf);
|
||||
};
|
||||
msgbuf_free(msgbuf);
|
||||
if((rv == 0) || (rv == -ENODEV))
|
||||
return;
|
||||
|
||||
_process_tp->enqueue_work(ptok,func);
|
||||
return ::_print_error(rv);
|
||||
}
|
||||
|
||||
_process_tp->enqueue_work(ptok,
|
||||
[=]()
|
||||
{
|
||||
_se->process_buf(_se,msgbuf);
|
||||
msgbuf_free(msgbuf);
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
|
@ -147,20 +143,25 @@ struct SyncWorker
|
|||
|
||||
msgbuf = msgbuf_alloc();
|
||||
|
||||
do
|
||||
while(true)
|
||||
{
|
||||
pthread_setcancelstate(PTHREAD_CANCEL_ENABLE,NULL);
|
||||
rv = _se->receive_buf(_se,msgbuf);
|
||||
pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,NULL);
|
||||
if(rv == 0)
|
||||
return;
|
||||
if(retriable_receive_error(rv))
|
||||
if(rv > 0)
|
||||
break;
|
||||
if(::_retriable_receive_error(rv))
|
||||
continue;
|
||||
if(fatal_receive_error(rv))
|
||||
return handle_receive_error(rv,msgbuf);
|
||||
} while(false);
|
||||
|
||||
msgbuf_free(msgbuf);
|
||||
if((rv == 0) || (rv == -ENODEV))
|
||||
return;
|
||||
|
||||
return ::_print_error(rv);
|
||||
}
|
||||
|
||||
_se->process_buf(_se,msgbuf);
|
||||
|
||||
msgbuf_free(msgbuf);
|
||||
}
|
||||
}
|
||||
|
|
@ -193,274 +194,66 @@ fuse_start_thread(pthread_t *thread_id,
|
|||
|
||||
static
|
||||
int
|
||||
calculate_thread_count(const int raw_thread_count_)
|
||||
_calculate_thread_count(const int raw_thread_count_)
|
||||
{
|
||||
int thread_count;
|
||||
int thread_count = 1;
|
||||
|
||||
thread_count = 4;
|
||||
if(raw_thread_count_ == 0)
|
||||
thread_count = std::thread::hardware_concurrency();
|
||||
{
|
||||
thread_count = std::thread::hardware_concurrency();
|
||||
thread_count = std::min(8,thread_count);
|
||||
}
|
||||
else if(raw_thread_count_ < 0)
|
||||
thread_count = (std::thread::hardware_concurrency() / -raw_thread_count_);
|
||||
{
|
||||
thread_count = (std::thread::hardware_concurrency() / -raw_thread_count_);
|
||||
thread_count = std::min(1,thread_count);
|
||||
}
|
||||
else if(raw_thread_count_ > 0)
|
||||
thread_count = raw_thread_count_;
|
||||
|
||||
if(thread_count <= 0)
|
||||
thread_count = 1;
|
||||
{
|
||||
thread_count = raw_thread_count_;
|
||||
}
|
||||
|
||||
return thread_count;
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
calculate_thread_counts(int *read_thread_count_,
|
||||
int *process_thread_count_,
|
||||
int *process_thread_queue_depth_)
|
||||
_calculate_thread_counts(int *read_thread_count_,
|
||||
int *process_thread_count_,
|
||||
int *process_thread_queue_depth_)
|
||||
{
|
||||
if((*read_thread_count_ == -1) && (*process_thread_count_ == -1))
|
||||
if((*read_thread_count_ == 0) && (*process_thread_count_ == -1))
|
||||
{
|
||||
int nproc;
|
||||
|
||||
nproc = std::thread::hardware_concurrency();
|
||||
*read_thread_count_ = std::min(8,nproc);
|
||||
}
|
||||
else if((*read_thread_count_ == 0) && (*process_thread_count_ == 0))
|
||||
{
|
||||
int nproc;
|
||||
|
||||
nproc = std::thread::hardware_concurrency();
|
||||
*read_thread_count_ = 2;
|
||||
*process_thread_count_ = std::max(2,(nproc - 2));
|
||||
*process_thread_count_ = std::min(8,*process_thread_count_);
|
||||
}
|
||||
else if((*read_thread_count_ == 0) && (*process_thread_count_ != -1))
|
||||
{
|
||||
*read_thread_count_ = 2;
|
||||
*process_thread_count_ = ::_calculate_thread_count(*process_thread_count_);
|
||||
}
|
||||
else
|
||||
{
|
||||
*read_thread_count_ = ::calculate_thread_count(*read_thread_count_);
|
||||
*read_thread_count_ = ::_calculate_thread_count(*read_thread_count_);
|
||||
if(*process_thread_count_ != -1)
|
||||
*process_thread_count_ = ::calculate_thread_count(*process_thread_count_);
|
||||
*process_thread_count_ = ::_calculate_thread_count(*process_thread_count_);
|
||||
}
|
||||
|
||||
if(*process_thread_queue_depth_ <= 0)
|
||||
*process_thread_queue_depth_ = *process_thread_count_;
|
||||
}
|
||||
*process_thread_queue_depth_ = 2;
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_R1L(const CPU::ThreadIdVec read_threads_)
|
||||
{
|
||||
CPU::CPUVec cpus;
|
||||
|
||||
cpus = CPU::cpus();
|
||||
if(cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,cpus.front());
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_R1P(const CPU::ThreadIdVec read_threads_)
|
||||
{
|
||||
CPU::Core2CPUsMap core2cpus;
|
||||
|
||||
core2cpus = CPU::core2cpus();
|
||||
if(core2cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_RP1L(const CPU::ThreadIdVec read_threads_,
|
||||
const CPU::ThreadIdVec process_threads_)
|
||||
{
|
||||
CPU::CPUVec cpus;
|
||||
|
||||
cpus = CPU::cpus();
|
||||
if(cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,cpus.front());
|
||||
for(auto const thread_id : process_threads_)
|
||||
CPU::setaffinity(thread_id,cpus.front());
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_RP1P(const CPU::ThreadIdVec read_threads_,
|
||||
const CPU::ThreadIdVec process_threads_)
|
||||
{
|
||||
CPU::Core2CPUsMap core2cpus;
|
||||
|
||||
core2cpus = CPU::core2cpus();
|
||||
if(core2cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
for(auto const thread_id : process_threads_)
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_R1LP1L(const std::vector<pthread_t> read_threads_,
|
||||
const std::vector<pthread_t> process_threads_)
|
||||
{
|
||||
CPU::CPUVec cpus;
|
||||
|
||||
cpus = CPU::cpus();
|
||||
if(cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,cpus.front());
|
||||
|
||||
for(auto const thread_id : process_threads_)
|
||||
CPU::setaffinity(thread_id,cpus.back());
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_R1PP1P(const std::vector<pthread_t> read_threads_,
|
||||
const std::vector<pthread_t> process_threads_)
|
||||
{
|
||||
CPU::Core2CPUsMap core2cpus;
|
||||
|
||||
core2cpus = CPU::core2cpus();
|
||||
if(core2cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
|
||||
if(core2cpus.size() > 1)
|
||||
core2cpus.erase(core2cpus.begin());
|
||||
|
||||
for(auto const thread_id : process_threads_)
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_RPSL(const std::vector<pthread_t> read_threads_,
|
||||
const std::vector<pthread_t> process_threads_)
|
||||
{
|
||||
CPU::CPUVec cpus;
|
||||
|
||||
cpus = CPU::cpus();
|
||||
if(cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
{
|
||||
if(cpus.empty())
|
||||
cpus = CPU::cpus();
|
||||
CPU::setaffinity(thread_id,cpus.back());
|
||||
cpus.pop_back();
|
||||
}
|
||||
|
||||
for(auto const thread_id : process_threads_)
|
||||
{
|
||||
if(cpus.empty())
|
||||
cpus = CPU::cpus();
|
||||
CPU::setaffinity(thread_id,cpus.back());
|
||||
cpus.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_RPSP(const std::vector<pthread_t> read_threads_,
|
||||
const std::vector<pthread_t> process_threads_)
|
||||
{
|
||||
CPU::Core2CPUsMap core2cpus;
|
||||
|
||||
core2cpus = CPU::core2cpus();
|
||||
if(core2cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
{
|
||||
if(core2cpus.empty())
|
||||
core2cpus = CPU::core2cpus();
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
core2cpus.erase(core2cpus.begin());
|
||||
}
|
||||
|
||||
for(auto const thread_id : process_threads_)
|
||||
{
|
||||
if(core2cpus.empty())
|
||||
core2cpus = CPU::core2cpus();
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
core2cpus.erase(core2cpus.begin());
|
||||
}
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads_R1PPSP(const std::vector<pthread_t> read_threads_,
|
||||
const std::vector<pthread_t> process_threads_)
|
||||
{
|
||||
CPU::Core2CPUsMap core2cpus;
|
||||
CPU::Core2CPUsMap leftover;
|
||||
|
||||
core2cpus = CPU::core2cpus();
|
||||
if(core2cpus.empty())
|
||||
return;
|
||||
|
||||
for(auto const thread_id : read_threads_)
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
|
||||
core2cpus.erase(core2cpus.begin());
|
||||
if(core2cpus.empty())
|
||||
core2cpus = CPU::core2cpus();
|
||||
leftover = core2cpus;
|
||||
|
||||
for(auto const thread_id : process_threads_)
|
||||
{
|
||||
if(core2cpus.empty())
|
||||
core2cpus = leftover;
|
||||
CPU::setaffinity(thread_id,core2cpus.begin()->second);
|
||||
core2cpus.erase(core2cpus.begin());
|
||||
}
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
pin_threads(const std::vector<pthread_t> read_threads_,
|
||||
const std::vector<pthread_t> process_threads_,
|
||||
const std::string type_)
|
||||
{
|
||||
if(type_.empty() || (type_ == "false"))
|
||||
return;
|
||||
if(type_ == "R1L")
|
||||
return ::pin_threads_R1L(read_threads_);
|
||||
if(type_ == "R1P")
|
||||
return ::pin_threads_R1P(read_threads_);
|
||||
if(type_ == "RP1L")
|
||||
return ::pin_threads_RP1L(read_threads_,process_threads_);
|
||||
if(type_ == "RP1P")
|
||||
return ::pin_threads_RP1P(read_threads_,process_threads_);
|
||||
if(type_ == "R1LP1L")
|
||||
return ::pin_threads_R1LP1L(read_threads_,process_threads_);
|
||||
if(type_ == "R1PP1P")
|
||||
return ::pin_threads_R1PP1P(read_threads_,process_threads_);
|
||||
if(type_ == "RPSL")
|
||||
return ::pin_threads_RPSL(read_threads_,process_threads_);
|
||||
if(type_ == "RPSP")
|
||||
return ::pin_threads_RPSP(read_threads_,process_threads_);
|
||||
if(type_ == "R1PPSP")
|
||||
return ::pin_threads_R1PPSP(read_threads_,process_threads_);
|
||||
|
||||
syslog(LOG_WARNING,
|
||||
"Invalid pin-threads value, ignoring: %s",
|
||||
type_.c_str());
|
||||
}
|
||||
|
||||
static
|
||||
void
|
||||
wait(fuse_session *se_,
|
||||
sem_t *finished_sem_)
|
||||
{
|
||||
while(!fuse_session_exited(se_))
|
||||
sem_wait(finished_sem_);
|
||||
*process_thread_queue_depth_ *= std::abs(*process_thread_count_);
|
||||
}
|
||||
|
||||
int
|
||||
|
|
@ -484,14 +277,13 @@ fuse_session_loop_mt(struct fuse_session *se_,
|
|||
read_thread_count = raw_read_thread_count_;
|
||||
process_thread_count = raw_process_thread_count_;
|
||||
process_thread_queue_depth = raw_process_thread_queue_depth_;
|
||||
::calculate_thread_counts(&read_thread_count,
|
||||
&process_thread_count,
|
||||
&process_thread_queue_depth);
|
||||
::_calculate_thread_counts(&read_thread_count,
|
||||
&process_thread_count,
|
||||
&process_thread_queue_depth);
|
||||
|
||||
if(process_thread_count > 0)
|
||||
process_tp = std::make_shared<ThreadPool>(process_thread_count,
|
||||
(process_thread_count *
|
||||
process_thread_queue_depth),
|
||||
process_thread_queue_depth,
|
||||
"fuse.process");
|
||||
|
||||
read_tp = std::make_unique<ThreadPool>(read_thread_count,
|
||||
|
|
@ -513,20 +305,19 @@ fuse_session_loop_mt(struct fuse_session *se_,
|
|||
if(process_tp)
|
||||
process_threads = process_tp->threads();
|
||||
|
||||
::pin_threads(read_threads,process_threads,pin_threads_type_);
|
||||
PinThreads::pin(read_threads,process_threads,pin_threads_type_);
|
||||
|
||||
syslog(LOG_INFO,
|
||||
"read-thread-count=%d; "
|
||||
"process-thread-count=%d; "
|
||||
"process-thread-queue-depth=%d; "
|
||||
"pin-threads=%s;"
|
||||
,
|
||||
read_thread_count,
|
||||
process_thread_count,
|
||||
process_thread_queue_depth,
|
||||
pin_threads_type_.c_str());
|
||||
SysLog::info("read-thread-count={}; "
|
||||
"process-thread-count={}; "
|
||||
"process-thread-queue-depth={}; "
|
||||
"pin-threads={};",
|
||||
read_thread_count,
|
||||
process_thread_count,
|
||||
process_thread_queue_depth,
|
||||
pin_threads_type_);
|
||||
|
||||
::wait(se_,&finished);
|
||||
while(!fuse_session_exited(se_))
|
||||
sem_wait(&finished);
|
||||
|
||||
sem_destroy(&finished);
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue