mergerfs/vendored/libfuse/include/thread_pool.hpp

/*
  ISC License

  Copyright (c) 2026, Antonio SJ Musumeci <trapexit@spawn.link>

  Permission to use, copy, modify, and/or distribute this software for any
  purpose with or without fee is hereby granted, provided that the above
  copyright notice and this permission notice appear in all copies.

  THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
*/

#pragma once

#include "bounded_queue.hpp"
#include "invocable.h"

#include "mutex.hpp"

#include <algorithm>
#include <atomic>
#include <cerrno>
#include <csignal>
#include <cstring>
#include <future>
#include <stdexcept>
#include <string>
#include <thread>
#include <utility>
#include <vector>

#include <pthread.h>
#include <syslog.h>
#include <time.h>


struct ThreadPoolTraits : public moodycamel::ConcurrentQueueDefaultTraits
{
  static const int MAX_SEMA_SPINS = 1;
};


// Shutdown contract:
//   Callers must stop submitting work before destroying the pool.
//   The destructor does not drain the queue: tasks enqueued but not
//   yet dequeued when the destructor runs are discarded.  For
//   enqueue_task this means the std::promise is destroyed unfulfilled
//   and the caller's future.get() throws std::future_error with
//   broken_promise.  enqueue_work* variants have no return value and
//   the work is silently dropped.
//
//   An enqueue racing with the destructor is otherwise safe — the
//   BoundedQueue destructor runs only after all workers are joined, so
//   the queue is not accessed concurrently during teardown.
//
//   add_thread(), remove_thread(), and set_threads() return
//   -ECANCELED when called while shutdown is in progress.  Treat a
//   non-zero return during teardown as expected, not as a failure.
class ThreadPool
{
public:
  using PToken = moodycamel::ProducerToken;
  using CToken = moodycamel::ConsumerToken;

private:
  using Func  = ofats::any_invocable<void()>;
  using Queue = BoundedQueue<Func,ThreadPoolTraits>;

public:
  struct ScalingConfig
  {
    std::size_t  min_threads           = 2;
    std::size_t  max_threads           = 64;
    std::int64_t sample_interval_usecs = 100000;   // 100ms hill-climb sample period
    std::int64_t cooldown_usecs        = 200000;   // 200ms min time between scaling events
    std::int64_t idle_threshold_usecs  = 5000000;  // 5s worker idle timeout before self-exit
    int          decline_threshold     = 2;        // consecutive declining samples before reversing direction
  };

  explicit
  ThreadPool(const unsigned     thread_count_      = std::thread::hardware_concurrency(),
             const unsigned     max_queue_depth_    = std::thread::hardware_concurrency() * 2,
             const std::string &name_               = {});
  explicit
  ThreadPool(const unsigned     thread_count_,
             const unsigned     max_queue_depth_,
             const std::string &name_,
             ScalingConfig      scaling_config_);
  ~ThreadPool();

  ThreadPool(const ThreadPool&) = delete;
  ThreadPool& operator=(const ThreadPool&) = delete;
  ThreadPool(ThreadPool&&) = delete;
  ThreadPool& operator=(ThreadPool&&) = delete;

  enum WorkerAction { CONTINUE, BREAK, EXIT };

private:
  static void *start_routine(void *arg_);
  static void *start_routine_autoscale(void *arg_);
  static void *monitor_routine(void *arg_);

  WorkerAction _process_func(Func &func_);
  int  _add_thread_scaling_locked(const std::string &name_);
  int  _remove_thread_scaling_locked();
  bool _try_remove_self_if_above(pthread_t t_, std::size_t min_count_);
  bool _try_claim_and_remove_self(pthread_t t_);
  void _maybe_grow_on_pressure();
  static std::uint64_t _now_usecs();
  static void _set_thread_name(pthread_t t_, const std::string &name_);

  std::atomic<std::size_t> _remove_count{0};

public:
  int add_thread(const std::string &name = {});
  int remove_thread();
  int set_threads(const std::size_t count);

public:
  template<typename FuncType>
  void
  enqueue_work(ThreadPool::PToken  &ptok_,
               FuncType           &&func_);

  template<typename FuncType>
  void
  enqueue_work(FuncType &&func_);

  template<typename FuncType>
  bool
  try_enqueue_work(ThreadPool::PToken &ptok_,
                   FuncType &&func_);

  template<typename FuncType>
  bool
  try_enqueue_work(FuncType &&func_);

  template<typename FuncType>
  bool
  try_enqueue_work_for(ThreadPool::PToken  &ptok_,
                       std::int64_t         timeout_usecs_,
                       FuncType           &&func_);

  template<typename FuncType>
  bool
  try_enqueue_work_for(std::int64_t  timeout_usecs_,
                       FuncType    &&func_);

  template<typename FuncType>
  [[nodiscard]]
  std::future<std::invoke_result_t<FuncType>>
  enqueue_task(FuncType&& func_);

  template<typename FuncType>
  [[nodiscard]]
  std::future<std::invoke_result_t<FuncType>>
  enqueue_task(ThreadPool::PToken &ptok_, FuncType&& func_);

  // Auto-scaling enqueue: grows pool on queue pressure
  template<typename FuncType>
  void
  enqueue_work_autoscale(ThreadPool::PToken  &ptok_,
                         FuncType           &&func_);

  template<typename FuncType>
  void
  enqueue_work_autoscale(FuncType &&func_);

public:
  std::vector<pthread_t> threads() const;
  ThreadPool::PToken ptoken();

  // Introspection
  // thread_count() reads an atomic mirror of _threads.size() and is
  // eventually consistent: at rest (no in-flight scaling operation)
  // the value equals _threads.size().  A caller that needs an exact
  // count must be sure no concurrent add/remove/set_threads is in
  // flight, or wait on the pool settling (see wait_until patterns in
  // tests/tests.cpp).
  std::size_t thread_count() const;
  std::size_t queue_depth() const;
  std::size_t queue_capacity() const;
  std::uint64_t tasks_enqueued() const;
  std::uint64_t tasks_completed() const;
  std::uint64_t tasks_pending() const;

private:
  Queue _queue;

private:
  const std::string      _name;
  std::vector<pthread_t> _threads;
  mutable Mutex          _threads_mutex;
  Mutex                  _scaling_mutex;
  std::atomic<bool>      _stop{false};

  // Mirror of _threads.size() maintained under _threads_mutex.  Lets
  // hot-path callers (_maybe_grow_on_pressure, monitor hints, public
  // thread_count() introspection) read the count without contending
  // on _threads_mutex.  Authoritative mutation paths still re-read
  // _threads.size() under the lock; the atomic is only a hint
  // externally, guaranteed-consistent at rest.
  std::atomic<std::size_t> _thread_count_atomic{0};

  // Monitoring counters
  std::atomic<std::uint64_t> _tasks_enqueued{0};
  std::atomic<std::uint64_t> _tasks_completed{0};

  // Scaling state
  // _dynamic_scaling_enabled is derived from min/max bounds and is
  // immutable after construction.  _scaling_config is fully immutable
  // after construction.  _monitor_started tracks whether the monitor
  // thread exists so the destructor knows whether it must wake/join it.
  bool                    _dynamic_scaling_enabled;
  const ScalingConfig     _scaling_config;
  pthread_t         _monitor_thread;
  bool              _monitor_started;
  // Raw pthreads (not Mutex/LockGuard) because the condvar requires
  // CLOCK_MONOTONIC via pthread_condattr_setclock, which the Mutex
  // wrapper doesn't support.
  pthread_mutex_t   _monitor_mutex;
  pthread_cond_t    _monitor_cond;

  std::atomic<std::uint64_t> _last_scale_time_usecs{0};
};


inline
std::uint64_t
ThreadPool::_now_usecs()
{
  struct timespec ts;
  clock_gettime(CLOCK_MONOTONIC,&ts);
  return static_cast<std::uint64_t>(ts.tv_sec) * 1000000ULL + static_cast<std::uint64_t>(ts.tv_nsec) / 1000ULL;
}


inline
void
ThreadPool::_set_thread_name(pthread_t            t_,
                             const std::string   &name_)
{
  if(name_.empty())
    return;
  std::string n = name_.substr(0,15);
  pthread_setname_np(t_,n.c_str());
}


inline
ThreadPool::ThreadPool(const unsigned        thread_count_,
                       const unsigned        max_queue_depth_,
                       const std::string    &name_)
  : ThreadPool(thread_count_,
               max_queue_depth_,
               name_,
               ScalingConfig{thread_count_, thread_count_})
{
}


inline
ThreadPool::ThreadPool(const unsigned        thread_count_,
                       const unsigned        max_queue_depth_,
                       const std::string    &name_,
                       ScalingConfig         scaling_config_)
  : _queue(max_queue_depth_ > 0
           ? max_queue_depth_
           : throw std::invalid_argument("threadpool: max_queue_depth must be > 0")),
    _name(name_),
    _dynamic_scaling_enabled(scaling_config_.min_threads < scaling_config_.max_threads),
    _scaling_config(std::move(scaling_config_)),
    _monitor_thread(),
    _monitor_started(false),
    _monitor_mutex(),
    _monitor_cond()
{
  // Initialize monitor mutex with pthread_mutex_init (not
  // PTHREAD_MUTEX_INITIALIZER which is for static/aggregate init).
  {
    int rv = pthread_mutex_init(&_monitor_mutex,NULL);
    if(rv != 0)
      throw std::runtime_error("threadpool: pthread_mutex_init failed");
  }

  // Use CLOCK_MONOTONIC for the monitor condvar so timed waits
  // are immune to wall-clock adjustments (NTP, settimeofday).
  {
    pthread_condattr_t attr;
    int rv;
    rv = pthread_condattr_init(&attr);
    if(rv != 0)
      {
        pthread_mutex_destroy(&_monitor_mutex);
        throw std::runtime_error("threadpool: pthread_condattr_init failed");
      }
    rv = pthread_condattr_setclock(&attr,CLOCK_MONOTONIC);
    if(rv != 0)
      {
        pthread_condattr_destroy(&attr);
        pthread_mutex_destroy(&_monitor_mutex);
        throw std::runtime_error("threadpool: pthread_condattr_setclock failed");
      }
    rv = pthread_cond_init(&_monitor_cond,&attr);
    pthread_condattr_destroy(&attr);
    if(rv != 0)
      {
        pthread_mutex_destroy(&_monitor_mutex);
        throw std::runtime_error("threadpool: pthread_cond_init failed");
      }
  }

  // Validate scaling config.  min_threads > max_threads is almost
  // certainly a caller mistake — silently disabling dynamic scaling
  // would hide the error.
  if(_scaling_config.min_threads > _scaling_config.max_threads)
    {
      pthread_cond_destroy(&_monitor_cond);
      pthread_mutex_destroy(&_monitor_mutex);
      throw std::invalid_argument(
        "threadpool: scaling_config.min_threads > max_threads");
    }

  // Timing values are only used when dynamic scaling is enabled
  // (monitor thread + autoscale workers).  Non-positive values cause
  // degenerate behavior:
  //   sample_interval_usecs <= 0:  monitor tight-spins (timedwait
  //       returns immediately with an in-the-past deadline)
  //   idle_threshold_usecs  <= 0:  workers spin-exit on every empty
  //       dequeue, draining the pool to min_threads
  //   cooldown_usecs        <= 0:  negative wraps to a huge unsigned
  //       in the comparison, disabling the cooldown gate
  if(_dynamic_scaling_enabled)
    {
      if(_scaling_config.sample_interval_usecs <= 0 ||
         _scaling_config.cooldown_usecs        <= 0 ||
         _scaling_config.idle_threshold_usecs  <= 0 ||
         _scaling_config.decline_threshold     <= 0)
        {
          pthread_cond_destroy(&_monitor_cond);
          pthread_mutex_destroy(&_monitor_mutex);
          throw std::invalid_argument(
            "threadpool: scaling_config timing/threshold values must be positive");
        }
    }

  // Clamp initial thread count to configured bounds.
  std::size_t count = thread_count_;
  if(count < _scaling_config.min_threads)
    count = _scaling_config.min_threads;
  if(count > _scaling_config.max_threads)
    count = _scaling_config.max_threads;

  sigset_t oldset;
  sigset_t newset;

  sigfillset(&newset);
  pthread_sigmask(SIG_BLOCK,&newset,&oldset);

  _threads.reserve(count);
  for(std::size_t i = 0; i < count; ++i)
    {
      int rv;
      pthread_t t;
      auto routine = _dynamic_scaling_enabled
        ? ThreadPool::start_routine_autoscale
        : ThreadPool::start_routine;

      rv = pthread_create(&t,NULL,routine,this);
      if(rv != 0)
        {
          syslog(LOG_WARNING,
                 "threadpool (%s): error spawning thread - %d (%s)",
                 _name.c_str(),
                 rv,
                 strerror(rv));
          continue;
        }

      _set_thread_name(t,_name);

      // Lock around push_back because a just-spawned autoscale
      // worker can time out on the empty queue and call
      // _try_remove_self_if_above (which acquires _threads_mutex
      // and modifies _threads) before the constructor finishes.
      // Concurrent unlocked push_back and locked erase is UB.
      {
        mutex_lockguard(_threads_mutex);
        _threads.push_back(t);
        _thread_count_atomic.store(_threads.size(),
                                   std::memory_order_release);
      }
    }

  pthread_sigmask(SIG_SETMASK,&oldset,NULL);

  // Check under lock: a spawned autoscale worker could have
  // already modified _threads via _try_remove_self_if_above.
  {
    std::size_t n;
    {
      mutex_lockguard(_threads_mutex);
      n = _threads.size();
    }

    if(n == 0)
      {
        pthread_cond_destroy(&_monitor_cond);
        pthread_mutex_destroy(&_monitor_mutex);
        throw std::runtime_error("threadpool: failed to spawn any threads");
      }

    syslog(LOG_DEBUG,
           "threadpool (%s): spawned %zu threads w/ max queue depth %u",
           _name.c_str(),
           n,
           max_queue_depth_);
  }

  // Start the monitor only when dynamic scaling can actually change
  // the thread count. Fixed-size pools still use _scaling_config for
  // their initial count, but don't need the monitor thread.
  if(_dynamic_scaling_enabled)
    {
      sigfillset(&newset);
      pthread_sigmask(SIG_BLOCK,&newset,&oldset);

      int rv = pthread_create(&_monitor_thread,NULL,
                              ThreadPool::monitor_routine,this);
      pthread_sigmask(SIG_SETMASK,&oldset,NULL);

      if(rv != 0)
        {
          syslog(LOG_WARNING,
                 "threadpool (%s): failed to spawn monitor thread - %d (%s); "
                 "continuing without hill-climb monitor",
                 _name.c_str(),
                 rv,
                 strerror(rv));
        }
      else
        {
          _monitor_started = true;
          {
            // Truncate _name to leave room for the ".mon" suffix
            // within the 15-char pthread_setname_np limit, so the
            // monitor thread is distinguishable from workers in ps/htop.
            std::string mon_name = _name.empty()
              ? "tp.monitor"
              : (_name.substr(0,11) + ".mon");
            _set_thread_name(_monitor_thread,mon_name);
          }
          syslog(LOG_DEBUG,
                 "threadpool (%s): auto-scaling enabled [%zu, %zu]",
                 _name.c_str(),
                 _scaling_config.min_threads,
                 _scaling_config.max_threads);
        }
    }
}


inline
ThreadPool::~ThreadPool()
{
  _stop.store(true,std::memory_order_release);

  // Wake the monitor thread so it exits immediately instead of
  // sleeping for up to one sample interval.
  if(_monitor_started)
    {
      pthread_mutex_lock(&_monitor_mutex);
      pthread_cond_signal(&_monitor_cond);
      pthread_mutex_unlock(&_monitor_mutex);
      pthread_join(_monitor_thread,NULL);
    }

  // Snapshot threads under both locks.  _scaling_mutex serializes
  // with in-flight _add_thread_scaling_locked calls so a thread
  // created by _maybe_grow_on_pressure (which passed the _stop
  // check before the store above) finishes its push_back before
  // we snapshot.  Without _scaling_mutex a late push_back could
  // land after the snapshot, leaving the new thread unjoined and
  // accessing a destroyed pool.
  // _threads_mutex closes the race with _try_remove_self_if_above and
  // _try_claim_and_remove_self — both check _stop under this lock, so
  // after _stop is set they become no-ops and leave us authoritative.
  std::vector<pthread_t> snapshot;
  {
    mutex_lockguard(_scaling_mutex);
    mutex_lockguard(_threads_mutex);
    snapshot = _threads;
  }

  syslog(LOG_DEBUG,
         "threadpool (%s): destroying %zu threads",
         _name.c_str(),
         snapshot.size());

  // Enqueue one poison pill per thread
  for(std::size_t i = 0; i < snapshot.size(); ++i)
    _queue.enqueue_unbounded(Func{});

  for(auto t : snapshot)
    pthread_join(t,NULL);

  // Destroy the monitor condvar/mutex only after all workers are
  // joined, so no thread can still reference them.
  pthread_cond_destroy(&_monitor_cond);
  pthread_mutex_destroy(&_monitor_mutex);
}


// Shared post-dequeue logic for both worker loops.
// Returns BREAK for poison pill, EXIT for _remove_count claim, CONTINUE otherwise.
inline
ThreadPool::WorkerAction
ThreadPool::_process_func(Func &func_)
{
  // Empty func is a poison pill — either a shutdown signal from
  // the destructor or a wakeup from _remove_thread_scaling_locked.
  if(!func_)
    {
      // During shutdown the destructor owns all joins; exit without detach.
      if(_stop.load(std::memory_order_acquire))
        return BREAK;

      // Try to claim a pending shrink request and atomically remove
      // self under both _scaling_mutex and _threads_mutex.  This
      // prevents set_threads from observing a stale effective count
      // (decremented _remove_count but _threads not yet shrunk) and
      // issuing excess removals that drain the pool to zero.
      if(_try_claim_and_remove_self(pthread_self()))
        return EXIT;

      return CONTINUE;
    }

  try
    {
      func_();
    }
  catch(const std::exception &e)
    {
      syslog(LOG_CRIT,
             "threadpool (%s): uncaught exception caught by worker - %s",
             _name.c_str(),
             e.what());
    }
  catch(...)
    {
      syslog(LOG_CRIT,
             "threadpool (%s): uncaught non-standard exception caught by worker",
             _name.c_str());
    }

  // Force destruction to release resources
  func_ = {};

  _tasks_completed.fetch_add(1,std::memory_order_relaxed);

  // _remove_thread_scaling_locked increments _remove_count.
  // Workers race to claim a decrement after each task; the winner
  // detaches itself and exits.  Uses the combined claim+remove to
  // prevent the TOCTOU race with set_threads.  During shutdown
  // (_stop set), _try_claim_and_remove_self returns false, so
  // the destructor still joins us.
  //
  // Fast path: most tasks complete with no shrink pending, so avoid
  // the double-lock acquisition in _try_claim_and_remove_self unless
  // _remove_count looks non-zero.  A stale zero read just defers the
  // claim by one task — a later completion on any worker picks it up.
  if(_remove_count.load(std::memory_order_relaxed) != 0 &&
     _try_claim_and_remove_self(pthread_self()))
    return EXIT;

  return CONTINUE;
}


// Standard worker loop: blocks indefinitely on dequeue.
inline
void*
ThreadPool::start_routine(void *arg_)
{
  ThreadPool *btp = static_cast<ThreadPool*>(arg_);

  ThreadPool::Func func;
  ThreadPool::Queue &q = btp->_queue;
  ThreadPool::CToken ctok(btp->_queue.make_ctoken());

  while(true)
    {
      q.wait_dequeue(ctok,func);

      WorkerAction action = btp->_process_func(func);
      if(action != CONTINUE)
        break;
    }

  return nullptr;
}


// Auto-scaling worker loop: uses timed dequeue so idle threads can self-exit.
inline
void*
ThreadPool::start_routine_autoscale(void *arg_)
{
  ThreadPool *btp = static_cast<ThreadPool*>(arg_);

  ThreadPool::Func func;
  ThreadPool::Queue &q = btp->_queue;
  ThreadPool::CToken ctok(btp->_queue.make_ctoken());
  const std::string name = btp->_name;
  const std::int64_t idle_threshold = btp->_scaling_config.idle_threshold_usecs;

  while(true)
    {
      bool got = q.try_wait_dequeue_for(ctok,func,
                                        idle_threshold);

      if(!got)
        {
          // Timed out - check if we should shrink
          if(btp->_stop.load(std::memory_order_acquire))
            break;

          // Snapshot the minimum thread count BEFORE calling
          // _try_remove_self_if_above.  If that method returns true
          // the thread is detached and removed from _threads — the
          // destructor's snapshot won't include it, so ~ThreadPool
          // may run concurrently on another thread.  Reading
          // btp->_scaling_config as an argument to the call would
          // dereference the pool object before the lock inside
          // _try_remove_self_if_above serializes with the destructor.
          //
          // name was already copied into a local at function entry,
          // so the syslog below is also safe.
          // Floor at 1 to match _remove_thread_scaling_locked's
          // min_allowed floor.  Without this, min_threads == 0
          // would let all workers self-exit, draining the pool
          // to zero threads and stalling non-autoscale enqueues.
          const std::size_t min_thr = std::max(btp->_scaling_config.min_threads,
                                               static_cast<std::size_t>(1));
          if(btp->_try_remove_self_if_above(pthread_self(),min_thr))
            {
              syslog(LOG_DEBUG,
                     "threadpool (%s): idle self-exit",
                     name.c_str());
              return nullptr;
            }

          continue;
        }

      WorkerAction action = btp->_process_func(func);
      if(action != CONTINUE)
        break;
    }

  return nullptr;
}


// Hill-climbing monitor thread.
// Periodically samples throughput, perturbs thread count by +/-1,
// and moves in the direction that improves throughput.
// Uses an EMA to smooth throughput noise and requires two consecutive
// declining samples before reversing direction.
inline
void*
ThreadPool::monitor_routine(void *arg_)
{
  ThreadPool *btp = static_cast<ThreadPool*>(arg_);

  const std::int64_t interval = btp->_scaling_config.sample_interval_usecs;
  int direction = 1;
  int decline_streak = 0;
  std::uint64_t prev_completed = btp->_tasks_completed.load(std::memory_order_relaxed);
  double ema_throughput = 0.0;
  constexpr double ema_alpha = 0.3;  // weight for new samples
  const int decline_threshold = btp->_scaling_config.decline_threshold;
  int warmup_samples = 0;

  // Track expected thread count after the last perturbation so we
  // can detect external scaling (fast-path grow, idle self-exit)
  // and avoid misattributing their throughput effects to our action.
  std::size_t expected_count = btp->thread_count();
  bool acted_last_cycle = false;

  while(true)
    {
      // Sleep for one sample interval, but wake immediately if the
      // destructor signals _monitor_cond.
      std::uint64_t t0 = _now_usecs();
      {
        struct timespec abs_ts;
        clock_gettime(CLOCK_MONOTONIC,&abs_ts);
        abs_ts.tv_nsec += (interval % 1000000) * 1000;
        abs_ts.tv_sec  += interval / 1000000;
        if(abs_ts.tv_nsec >= 1000000000L)
          {
            abs_ts.tv_sec  += 1;
            abs_ts.tv_nsec -= 1000000000L;
          }

        // Check _stop under _monitor_mutex to close the lost-wakeup
        // race: without this, the destructor could set _stop and
        // signal between our check and the timedwait, losing the
        // signal and delaying shutdown by up to one sample interval.
        pthread_mutex_lock(&btp->_monitor_mutex);
        if(btp->_stop.load(std::memory_order_acquire))
          {
            pthread_mutex_unlock(&btp->_monitor_mutex);
            break;
          }
        pthread_cond_timedwait(&btp->_monitor_cond,
                               &btp->_monitor_mutex,
                               &abs_ts);
        pthread_mutex_unlock(&btp->_monitor_mutex);
      }

      if(btp->_stop.load(std::memory_order_acquire))
        break;

      // Skip spurious wakeups.  A short interval yields an
      // artificially low throughput delta that would cause the EMA
      // to decline and potentially trigger a false direction
      // reversal in the hill-climber.  Reset prev_completed so the
      // next real cycle doesn't accumulate the skipped interval's
      // completions into a single inflated throughput sample.
      std::uint64_t elapsed = _now_usecs() - t0;
      if(elapsed < static_cast<std::uint64_t>(interval) / 2)
        {
          prev_completed = btp->_tasks_completed.load(std::memory_order_relaxed);
          continue;
        }

      std::uint64_t completed = btp->_tasks_completed.load(std::memory_order_relaxed);
      std::uint64_t delta = completed - prev_completed;
      prev_completed = completed;

      // Normalize to a rate (tasks per sample interval) so the
      // hill-climber isn't distorted by variable elapsed times.
      // The spurious-wakeup filter above accepts any cycle ≥ 50%
      // of the target interval, so raw counts can vary by up to
      // 2x from timing alone.  Normalizing removes that noise.
      double throughput_rate = static_cast<double>(delta) *
        (static_cast<double>(interval) / static_cast<double>(elapsed));

      // Warm up: seed the EMA with the first two samples before acting.
      if(warmup_samples < 2)
        {
          ema_throughput = (warmup_samples == 0)
            ? throughput_rate
            : ema_alpha * throughput_rate + (1.0 - ema_alpha) * ema_throughput;
          ++warmup_samples;
          continue;
        }

      double prev_ema = ema_throughput;
      ema_throughput = ema_alpha * throughput_rate + (1.0 - ema_alpha) * ema_throughput;

      // Only adjust if there's meaningful work happening.
      // Reset direction to +1 (bias toward growth) so a stale
      // shrink direction from before the idle period doesn't
      // penalize the pool when work resumes.  Also reset warmup
      // so the EMA re-seeds from fresh samples rather than
      // comparing against a stale pre-idle value.
      if(ema_throughput < 0.5 && prev_ema < 0.5)
        {
          decline_streak = 0;
          direction = 1;
          warmup_samples = 0;
          prev_completed = completed;
          continue;
        }

      std::size_t count = btp->thread_count();

      // If external scaling changed the thread count since our last
      // perturbation, we can't attribute the throughput delta to our
      // action.  Reset the streak and skip this cycle.
      if(acted_last_cycle && count != expected_count)
        {
          decline_streak = 0;
          acted_last_cycle = false;
          expected_count = count;
          prev_completed = completed;
          continue;
        }
      acted_last_cycle = false;

      // Hill-climb: did smoothed throughput improve since last sample?
      // Strict > so flat throughput (saturated/oversized pool) counts
      // as no improvement, allowing the hill-climber to eventually
      // shrink an oversized pool rather than only relying on idle timeout.
      if(ema_throughput > prev_ema)
        {
          decline_streak = 0;
        }
      else
        {
          ++decline_streak;
          if(decline_streak >= decline_threshold)
            {
              direction = -direction;
              decline_streak = 0;
            }
          else
            {
              continue;  // wait for confirmation before reversing
            }
        }

      // Skip perturbation when already at a bound.  Without this,
      // at max_threads the grow is a no-op but the next reversal
      // succeeds at shrinking, creating a slow downward leak.
      // Symmetrically at min_threads a shrink no-op followed by a
      // grow reversal would needlessly inflate the pool.
      if(direction > 0 && count >= btp->_scaling_config.max_threads)
        continue;
      if(direction < 0 && count <= btp->_scaling_config.min_threads)
        continue;

      // Hold _scaling_mutex across the cooldown check and the scaling
      // action so the monitor and fast-path pressure grow cannot both
      // pass the cooldown gate simultaneously.
      {
        mutex_lockguard(btp->_scaling_mutex);

        std::uint64_t now  = _now_usecs();
        std::uint64_t last = btp->_last_scale_time_usecs.load(std::memory_order_relaxed);
        if((now - last) < static_cast<std::uint64_t>(btp->_scaling_config.cooldown_usecs))
          continue;

        // Re-read thread count under _scaling_mutex so the bound
        // check is authoritative.  The earlier unlocked read is a
        // pre-lock hint used only by the skip-at-bounds check above;
        // by the time we get here an idle self-exit or fast-path grow
        // may have changed it.
        count = btp->thread_count();

        if(direction > 0 && count < btp->_scaling_config.max_threads)
          {
            if(btp->_add_thread_scaling_locked({}) == 0)
              {
                btp->_last_scale_time_usecs.store(now,
                                                  std::memory_order_relaxed);
                acted_last_cycle = true;
                // Use count + 1 rather than thread_count() so the
                // expected value reflects the intended outcome of our
                // action, not the racy current state.  For grow this
                // is equivalent (push_back is synchronous) but for
                // consistency and to avoid an extra lock acquisition,
                // compute it arithmetically.
                expected_count = count + 1;
                syslog(LOG_DEBUG,
                       "threadpool (%s): hill-climb grow to %zu threads "
                       "(ema %.1f -> %.1f)",
                       btp->_name.c_str(),
                       expected_count,
                       prev_ema,
                       ema_throughput);
              }
          }
        else if(direction < 0 && count > btp->_scaling_config.min_threads)
          {
            if(btp->_remove_thread_scaling_locked() == 0)
              {
                btp->_last_scale_time_usecs.store(now,
                                                  std::memory_order_relaxed);
                acted_last_cycle = true;
                // _remove_thread_scaling_locked only increments
                // _remove_count and enqueues a wakeup pill — it does
                // not synchronously erase from _threads.  Using
                // thread_count() here would return the pre-shrink
                // count, causing the next cycle to misattribute the
                // deferred removal to external scaling interference.
                expected_count = count - 1;
                syslog(LOG_DEBUG,
                       "threadpool (%s): hill-climb shrink to %zu threads "
                       "(ema %.1f -> %.1f)",
                       btp->_name.c_str(),
                       expected_count,
                       prev_ema,
                       ema_throughput);
              }
          }
      }
    }

  return nullptr;
}


// Atomically claim a pending removal AND remove self from the thread
// list under both _scaling_mutex and _threads_mutex.  Atomicity closes
// a TOCTOU race: without it, set_threads could observe the decremented
// _remove_count before _threads is shrunk, miscalculate effective
// count, and issue excess removals that drain the pool to zero.
inline
bool
ThreadPool::_try_claim_and_remove_self(pthread_t t_)
{
  mutex_lockguard(_scaling_mutex);
  mutex_lockguard(_threads_mutex);

  if(_stop.load(std::memory_order_acquire))
    return false;

  // Try to claim a pending removal.
  std::size_t count = _remove_count.load(std::memory_order_relaxed);
  bool claimed = false;
  while(count > 0)
    {
      if(_remove_count.compare_exchange_weak(count,count - 1,
                                              std::memory_order_acq_rel))
        {
          claimed = true;
          break;
        }
    }
  if(!claimed)
    return false;

  // Safety floor: never remove the last thread.  Use the same
  // min_allowed logic as _remove_thread_scaling_locked.
  std::size_t min_allowed = 1;
  if(_dynamic_scaling_enabled && _scaling_config.min_threads > min_allowed)
    min_allowed = _scaling_config.min_threads;
  if(_threads.size() <= min_allowed)
    {
      // Restore the claim — a future set_threads or grow will
      // either cancel it or the pool will be large enough.
      _remove_count.fetch_add(1,std::memory_order_relaxed);
      return false;
    }

  auto it = std::find(_threads.begin(),_threads.end(),t_);
  if(it == _threads.end())
    {
      _remove_count.fetch_add(1,std::memory_order_relaxed);
      return false;
    }

  _threads.erase(it);
  _thread_count_atomic.store(_threads.size(),
                             std::memory_order_release);
  pthread_detach(t_);
  return true;
}


// Atomically check thread count and remove self if above min_count_.
// Returns true if removed, false if at or below threshold.
//
// Acquires _scaling_mutex then _threads_mutex (matching the lock
// ordering used by _maybe_grow_on_pressure and the monitor) so that
// idle self-exit is serialized with set_threads, _remove_thread, and
// _add_thread.  Without _scaling_mutex, an idle self-exit racing with
// set_threads could cause set_threads to overshoot because it reads
// _threads.size() under _threads_mutex but the idle exit shrinks the
// pool outside _scaling_mutex.
inline
bool
ThreadPool::_try_remove_self_if_above(pthread_t    t_,
                                      std::size_t  min_count_)
{
  mutex_lockguard(_scaling_mutex);
  mutex_lockguard(_threads_mutex);

  // Check _stop under the lock to close the TOCTOU race with the
  // destructor.  The destructor sets _stop, then acquires
  // _threads_mutex to snapshot _threads.  By checking _stop here
  // under the same lock, either:
  //   (a) we hold the lock first, see _stop==false, detach and
  //       erase ourselves — the destructor's snapshot won't
  //       include us, so it won't join a detached thread; or
  //   (b) the destructor sets _stop first — we see _stop==true
  //       and bail out, remaining in _threads for the destructor
  //       to join.
  if(_stop.load(std::memory_order_acquire))
    return false;

  // Account for pending removals that haven't been claimed yet.
  // Without this, an idle self-exit can race with a monitor shrink:
  // the monitor increments _remove_count (a pending removal), and
  // the idle thread also removes itself, pushing the pool below
  // min_count_.
  std::size_t pending = _remove_count.load(std::memory_order_relaxed);
  std::size_t effective = (_threads.size() > pending)
    ? (_threads.size() - pending)
    : 0;
  if(effective <= min_count_)
    return false;

  auto it = std::find(_threads.begin(),_threads.end(),t_);
  if(it == _threads.end())
    return false;

  _threads.erase(it);
  _thread_count_atomic.store(_threads.size(),
                             std::memory_order_release);
  pthread_detach(t_);
  return true;
}


inline
void
ThreadPool::_maybe_grow_on_pressure()
{
  if(!_dynamic_scaling_enabled)
    return;

  // Don't spawn threads if the pool is shutting down.  Without this
  // check a caller could grow the pool after the destructor has
  // already sized its poison-pill batch, leaving the new thread
  // blocking forever on wait_dequeue with no pill to wake it.
  if(_stop.load(std::memory_order_acquire))
    return;

  // Quick checks before contending on the lock.  Uses stale reads
  // but the authoritative re-checks happen under the lock below.
  {
    std::uint64_t now  = _now_usecs();
    std::uint64_t last = _last_scale_time_usecs.load(std::memory_order_relaxed);
    if((now - last) < static_cast<std::uint64_t>(_scaling_config.cooldown_usecs))
      return;
  }
  if(thread_count() >= _scaling_config.max_threads)
    return;

  // Hold _scaling_mutex so the thread-count check and add are atomic
  // with respect to other scaling operations.
  mutex_lockguard(_scaling_mutex);

  std::uint64_t now  = _now_usecs();
  std::uint64_t last = _last_scale_time_usecs.load(std::memory_order_relaxed);
  if((now - last) < static_cast<std::uint64_t>(_scaling_config.cooldown_usecs))
    return;

  if(_add_thread_scaling_locked({}) != 0)
    return;
  _last_scale_time_usecs.store(now,std::memory_order_relaxed);

  syslog(LOG_DEBUG,
         "threadpool (%s): fast-path grow to %zu threads (queue full)",
         _name.c_str(),
         thread_count());
}


inline
int
ThreadPool::_add_thread_scaling_locked(const std::string &name_)
{
  // Don't spawn threads after the destructor has set _stop.
  // Combined with the destructor holding _scaling_mutex during
  // its snapshot, this ensures every created thread is joined.
  if(_stop.load(std::memory_order_acquire))
    return -ECANCELED;

  if(_dynamic_scaling_enabled)
    {
      mutex_lockguard(_threads_mutex);
      // Use effective count (raw minus pending removals) to match
      // _remove_thread_scaling_locked's symmetric min-bound check.
      // Without this, pending removal claims block the pool from
      // growing back to max_threads under load spikes.
      std::size_t pending = _remove_count.load(std::memory_order_relaxed);
      std::size_t effective = (_threads.size() > pending)
        ? (_threads.size() - pending)
        : 0;
      if(effective >= _scaling_config.max_threads)
        return -EAGAIN;
    }

  int rv;
  pthread_t t;
  sigset_t oldset;
  sigset_t newset;
  std::string name;

  name = (name_.empty() ? _name : name_);

  auto routine = _dynamic_scaling_enabled
    ? ThreadPool::start_routine_autoscale
    : ThreadPool::start_routine;

  sigfillset(&newset);
  pthread_sigmask(SIG_BLOCK,&newset,&oldset);
  rv = pthread_create(&t,NULL,routine,this);
  pthread_sigmask(SIG_SETMASK,&oldset,NULL);

  if(rv != 0)
    {
      syslog(LOG_WARNING,
             "threadpool (%s): error spawning thread - %d (%s)",
             _name.c_str(),
             rv,
             strerror(rv));

      // Treat creation failure as a scaling event so the cooldown
      // timer suppresses immediate retries from the monitor and
      // fast-path pressure grow, avoiding syslog spam under
      // sustained resource pressure.
      if(_dynamic_scaling_enabled)
        _last_scale_time_usecs.store(_now_usecs(),std::memory_order_relaxed);

      return -rv;
    }

  _set_thread_name(t,name);

  {
    mutex_lockguard(_threads_mutex);
    _threads.push_back(t);
    _thread_count_atomic.store(_threads.size(),
                               std::memory_order_release);
  }

  return 0;
}

inline
int
ThreadPool::add_thread(const std::string &name_)
{
  mutex_lockguard(_scaling_mutex);
  return _add_thread_scaling_locked(name_);
}


inline
int
ThreadPool::_remove_thread_scaling_locked()
{
  // Skip once shutdown has begun.  Mirrors _add_thread_scaling_locked's
  // early return for symmetry and prevents the monitor from enqueuing
  // wakeup pills that no worker will claim (workers see _stop and
  // BREAK before reaching _try_claim_and_remove_self), which would
  // leave stale pills in the queue until BoundedQueue is destroyed.
  if(_stop.load(std::memory_order_acquire))
    return -ECANCELED;

  // Hold _threads_mutex across both the size check and the
  // _remove_count increment so that a concurrent removal (idle
  // self-exit or another _remove_thread call) cannot shrink the
  // pool below min_allowed between the check and the signal.
  {
    mutex_lockguard(_threads_mutex);
    std::size_t min_allowed = 1;
    if(_dynamic_scaling_enabled && _scaling_config.min_threads > min_allowed)
      min_allowed = _scaling_config.min_threads;
    std::size_t pending = _remove_count.load(std::memory_order_relaxed);
    std::size_t effective = (_threads.size() > pending)
      ? (_threads.size() - pending)
      : 0;
    if(effective <= min_allowed)
      return -EINVAL;

    // Increment the atomic counter while still holding the lock.
    // A worker will claim the decrement either:
    //   (a) in the poison-pill handler after dequeuing the wakeup, or
    //   (b) in the task-completion CAS loop in _process_func.
    // In case (b) the wakeup pill becomes stale and is harmlessly
    // ignored (the pill handler sees _remove_count == 0 and continues).
    _remove_count.fetch_add(1,std::memory_order_relaxed);
  }

  _queue.enqueue_unbounded(Func{});

  return 0;
}

inline
int
ThreadPool::remove_thread()
{
  mutex_lockguard(_scaling_mutex);
  return _remove_thread_scaling_locked();
}

inline
int
ThreadPool::set_threads(const std::size_t count_)
{
  mutex_lockguard(_scaling_mutex);

  if(count_ == 0)
    return -EINVAL;

  // Clamp to autoscale bounds so external callers can't push the
  // pool outside the range the monitor and idle-exit logic expect.
  std::size_t count = count_;
  if(_dynamic_scaling_enabled)
    {
      if(count < _scaling_config.min_threads)
        count = _scaling_config.min_threads;
      if(count > _scaling_config.max_threads)
        count = _scaling_config.max_threads;
    }

  std::size_t current;
  std::size_t pending;

  {
    mutex_lockguard(_threads_mutex);
    current = _threads.size();
    // Read _remove_count under the same lock to get a consistent
    // snapshot.  Without the lock a worker could claim a
    // _remove_count decrement and erase itself from _threads
    // between the two reads, causing set_threads to double-count
    // the shrink and over-remove.
    pending = _remove_count.load(std::memory_order_relaxed);
  }

  std::size_t effective = (current > pending) ? (current - pending) : 0;

  int rv = 0;
  bool changed = false;

  for(std::size_t i = effective; i < count; ++i)
    {
      int err = _add_thread_scaling_locked({});
      if(err != 0)
        {
          rv = err;
          break;
        }
      changed = true;
    }

  for(std::size_t i = count; i < effective; ++i)
    {
      int err = _remove_thread_scaling_locked();
      if(err != 0)
        {
          rv = err;
          break;
        }
      changed = true;
    }

  // Stamp the cooldown timer so the monitor doesn't immediately
  // perturb the thread count after an explicit set_threads call.
  if(changed && _dynamic_scaling_enabled)
    _last_scale_time_usecs.store(_now_usecs(),std::memory_order_relaxed);

  return rv;
}

template<typename FuncType>
inline
void
ThreadPool::enqueue_work(ThreadPool::PToken  &ptok_,
                         FuncType           &&func_)
{
  _queue.enqueue(ptok_,
                 std::forward<FuncType>(func_));
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
}

template<typename FuncType>
inline
void
ThreadPool::enqueue_work(FuncType &&func_)
{
  _queue.enqueue(std::forward<FuncType>(func_));
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
}


template<typename FuncType>
inline
bool
ThreadPool::try_enqueue_work(ThreadPool::PToken &ptok_,
                             FuncType &&func_)
{
  if(!_queue.try_enqueue(ptok_,std::forward<FuncType>(func_)))
    return false;
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
  return true;
}


template<typename FuncType>
inline
bool
ThreadPool::try_enqueue_work(FuncType &&func_)
{
  if(!_queue.try_enqueue(std::forward<FuncType>(func_)))
    return false;
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
  return true;
}


template<typename FuncType>
inline
bool
ThreadPool::try_enqueue_work_for(ThreadPool::PToken  &ptok_,
                                 std::int64_t         timeout_usecs_,
                                 FuncType           &&func_)
{
  if(!_queue.try_enqueue_for(ptok_,timeout_usecs_,std::forward<FuncType>(func_)))
    return false;
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
  return true;
}


template<typename FuncType>
inline
bool
ThreadPool::try_enqueue_work_for(std::int64_t  timeout_usecs_,
                                 FuncType    &&func_)
{
  if(!_queue.try_enqueue_for(timeout_usecs_,std::forward<FuncType>(func_)))
    return false;
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
  return true;
}


template<typename FuncType>
inline
void
ThreadPool::enqueue_work_autoscale(ThreadPool::PToken  &ptok_,
                                   FuncType           &&func_)
{
  // Materialize into a Func up front so we own the value across
  // the try/block-enqueue boundary without a double std::forward.
  //
  // INVARIANT: BoundedQueue::try_enqueue checks the semaphore
  // (tryWait) *before* forwarding the item into the inner queue.
  // On failure the item is never moved-from, so f remains valid.
  // The check below guards against future BoundedQueue changes
  // that could violate this ordering.
  Func f(std::forward<FuncType>(func_));

  if(_queue.try_enqueue(ptok_,std::move(f)))
    {
      _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
      return;
    }

  // If try_enqueue moved the callable despite returning false, the
  // Func is empty and would act as a poison pill — re-enqueuing it
  // could cause a worker to exit.  Throw so the caller notices
  // immediately rather than silently corrupting the pool.
  if(!static_cast<bool>(f))
    throw std::logic_error("BoundedQueue::try_enqueue moved the callable despite returning false");

  _maybe_grow_on_pressure();

  // Block-enqueue (will succeed once a worker frees a slot)
  _queue.enqueue(ptok_,std::move(f));
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
}


template<typename FuncType>
inline
void
ThreadPool::enqueue_work_autoscale(FuncType &&func_)
{
  // See ptok_ overload for invariant and safety note.
  Func f(std::forward<FuncType>(func_));

  if(_queue.try_enqueue(std::move(f)))
    {
      _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
      return;
    }

  if(!static_cast<bool>(f))
    throw std::logic_error("BoundedQueue::try_enqueue moved the callable despite returning false");

  _maybe_grow_on_pressure();

  // Block-enqueue (will succeed once a worker frees a slot)
  _queue.enqueue(std::move(f));
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);
}


template<typename FuncType>
[[nodiscard]]
inline
std::future<std::invoke_result_t<FuncType>>
ThreadPool::enqueue_task(FuncType&& func_)
{
  using TaskReturnType = std::invoke_result_t<FuncType>;
  using Promise        = std::promise<TaskReturnType>;

  Promise promise;
  auto    future = promise.get_future();

  auto work =
    [promise_ = std::move(promise),
     func_    = std::forward<FuncType>(func_)]() mutable
    {
      try
        {
          if constexpr (std::is_void_v<TaskReturnType>)
            {
              func_();
              promise_.set_value();
            }
          else
            {
              promise_.set_value(func_());
            }
        }
      catch(...)
        {
          promise_.set_exception(std::current_exception());
        }
    };

  Func f(std::move(work));

  if(_dynamic_scaling_enabled)
    {
      if(!_queue.try_enqueue(std::move(f)))
        {
          if(!static_cast<bool>(f))
            throw std::logic_error("BoundedQueue::try_enqueue moved the callable despite returning false");
          _maybe_grow_on_pressure();
          _queue.enqueue(std::move(f));
        }
    }
  else
    {
      _queue.enqueue(std::move(f));
    }
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);

  return future;
}


template<typename FuncType>
[[nodiscard]]
inline
std::future<std::invoke_result_t<FuncType>>
ThreadPool::enqueue_task(ThreadPool::PToken &ptok_,
                         FuncType          &&func_)
{
  using TaskReturnType = std::invoke_result_t<FuncType>;
  using Promise        = std::promise<TaskReturnType>;

  Promise promise;
  auto    future = promise.get_future();

  auto work =
    [promise_ = std::move(promise),
     func_    = std::forward<FuncType>(func_)]() mutable
    {
      try
        {
          if constexpr (std::is_void_v<TaskReturnType>)
            {
              func_();
              promise_.set_value();
            }
          else
            {
              promise_.set_value(func_());
            }
        }
      catch(...)
        {
          promise_.set_exception(std::current_exception());
        }
    };

  Func f(std::move(work));

  if(_dynamic_scaling_enabled)
    {
      if(!_queue.try_enqueue(ptok_,std::move(f)))
        {
          if(!static_cast<bool>(f))
            throw std::logic_error("BoundedQueue::try_enqueue moved the callable despite returning false");
          _maybe_grow_on_pressure();
          _queue.enqueue(ptok_,std::move(f));
        }
    }
  else
    {
      _queue.enqueue(ptok_,std::move(f));
    }
  _tasks_enqueued.fetch_add(1,std::memory_order_relaxed);

  return future;
}


inline
std::vector<pthread_t>
ThreadPool::threads() const
{
  mutex_lockguard(_threads_mutex);

  return _threads;
}


inline
ThreadPool::PToken
ThreadPool::ptoken()
{
  return _queue.make_ptoken();
}


inline
std::size_t
ThreadPool::thread_count() const
{
  // Read from the atomic mirror rather than locking _threads_mutex.
  // Eventually consistent: at rest the value equals _threads.size();
  // in the tiny window between a vector mutation and the atomic store
  // (both under _threads_mutex), readers may observe the prior value.
  // Acceptable for every caller — _add/_remove_thread_scaling_locked
  // re-check under _threads_mutex, fast-path grow treats it as a
  // hint, and external introspection is documented as approximate.
  return _thread_count_atomic.load(std::memory_order_acquire);
}

inline
std::size_t
ThreadPool::queue_depth() const
{
  return _queue.size_approx();
}

inline
std::size_t
ThreadPool::queue_capacity() const
{
  return _queue.max_depth();
}

inline
std::uint64_t
ThreadPool::tasks_enqueued() const
{
  return _tasks_enqueued.load(std::memory_order_relaxed);
}

inline
std::uint64_t
ThreadPool::tasks_completed() const
{
  return _tasks_completed.load(std::memory_order_relaxed);
}

// Approximate: loads are atomic but the pair is not.
// Loading enqueued first is meant to bias the result toward zero (a
// completion between the two loads increases c beyond the snapshot
// of e, shrinking the delta), which is safer for scaling heuristics
// than over-reporting pressure.  Acquire ordering on both loads plus
// the intervening acquire fence prevents weak-memory hardware (ARM,
// POWER) from reordering them and defeating the intended bias.
// Suitable for monitoring and heuristics, not for synchronization.
inline
std::uint64_t
ThreadPool::tasks_pending() const
{
  auto e = _tasks_enqueued.load(std::memory_order_acquire);
  std::atomic_thread_fence(std::memory_order_acquire);
  auto c = _tasks_completed.load(std::memory_order_acquire);
  return (e > c) ? (e - c) : 0;
}