From c8f32f9f2590c12c0a72ac087937035523a6b559 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Fri, 8 May 2026 15:55:55 -0500 Subject: [PATCH 01/23] update build system and dependency configuration --- Makefile | 47 +++++++++++++---------- buildtools/build-release | 2 +- buildtools/containerfiles/fedora:44.amd64 | 2 +- buildtools/containerfiles/fedora:44.arm64 | 2 +- debian/rules | 5 ++- vendored/libfuse/Makefile | 36 ++++++++++------- 6 files changed, 56 insertions(+), 38 deletions(-) diff --git a/Makefile b/Makefile index e1cb8e58..4dc0977a 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ TOUCH ?= touch BUILDDIR := build GITHUB_REPO ?= "https://github.com/trapexit/mergerfs" -DEFAULT_RELEASE := debian:13.amd64 +DEFAULT_TARGET := debian:13.amd64 ifndef GIT_REPO ifneq ($(shell $(GIT) --version 2> /dev/null),) @@ -48,11 +48,7 @@ endif endif USE_XATTR ?= 1 -UGID_USE_RWLOCK ?= 0 -ifdef NDEBUG -OPT_FLAGS := -O2 -DNDEBUG -else ifdef SANITIZE ifeq ($(SANITIZE),1) override SANITIZE := -fsanitize=address,undefined,leak @@ -60,14 +56,22 @@ else ifeq ($(filter -fsanitize=%,$(SANITIZE)),) override SANITIZE := -fsanitize=$(SANITIZE) endif endif -OPT_FLAGS := -O0 \ - -g \ - -fno-omit-frame-pointer \ - $(SANITIZE) \ - -fstack-protector-strong \ - -Wextra \ - -Wno-unused-parameter \ - -DDEBUG + +COMMON_OPT_FLAGS += \ + -fstack-protector-strong \ + -Wextra \ + -Wno-unused-parameter + +DEBUG_OPT_FLAGS += \ + -O0 \ + -g3 \ + -fno-omit-frame-pointer \ + $(SANITIZE) + +ifdef RELEASE +OPT_FLAGS := -O3 $(COMMON_OPT_FLAGS) +else +OPT_FLAGS ?= $(DEBUG_OPT_FLAGS) $(COMMON_OPT_FLAGS) endif ifdef STATIC @@ -125,8 +129,7 @@ override INC_FLAGS := \ -Ivendored \ -Ivendored/libfuse/include override MFS_FLAGS := \ - -DUSE_XATTR=$(USE_XATTR) \ - -DUGID_USE_RWLOCK=$(UGID_USE_RWLOCK) + -DUSE_XATTR=$(USE_XATTR) override TESTS_FLAGS := \ -Isrc \ -Ivendored \ @@ -198,7 +201,7 @@ libfuse: $(LIBFUSE) $(MAKE) -C vendored/libfuse $(LIBFUSE): - $(MAKE) -C vendored/libfuse $(BUILDDIR)/libfuse.a + $(MAKE) -C vendored/libfuse $(BUILDDIR)/libfuse.a RELEASE=$(RELEASE) tests: $(BUILDDIR)/tests @@ -362,14 +365,16 @@ define build_release --branch=$(GITREF) endef -.PHONY: release +.PHONY: release release-all .PHONY: release-amd64 release-arm64 release-armhf release-riscv64 -.PHONY: release-sample release-static release-tarball +.PHONY: release-sample release-static release-tarball release-installer release: + $(MAKE) RELEASE=1 CPPFLAGS=-DNDEBUG all +release-all: $(call build_release,"all") release-sample: - $(if $(RELEASE),,$(eval RELEASE := $(shell ./buildtools/detect-local-target 2>/dev/null))) - $(call build_release,$(if $(RELEASE),$(RELEASE),$(DEFAULT_RELEASE))) + $(if $(TARGET),,$(eval TARGET := $(shell ./buildtools/detect-local-target 2>/dev/null))) + $(call build_release,$(if $(TARGET),$(TARGET),$(DEFAULT_TARGET))) release-amd64: $(call build_release,"amd64") release-arm64: @@ -382,6 +387,8 @@ release-static: $(call build_release,"static") release-tarball: $(call build_release,"tarball") +release-installer: + $(call build_release,"installer") container: $(eval GITREF ?= $(shell git describe --exact-match --tags HEAD 2>/dev/null || git symbolic-ref --short HEAD 2>/dev/null || git rev-parse --short HEAD)) diff --git a/buildtools/build-release b/buildtools/build-release index 6c307f51..241aeece 100755 --- a/buildtools/build-release +++ b/buildtools/build-release @@ -18,7 +18,7 @@ def build(git_repo, '--force-rm'] if os.path.exists(git_repo): git_repo = os.path.realpath(git_repo) - args += ['-v',f'{git_repo}:/tmp/mergerfs.git:ro'] + args += ['-v',f'{git_repo}:/tmp/mergerfs.git:ro:z'] args += ['--build-arg=GIT_REPO=file:///tmp/mergerfs.git'] else: args += [f'--build-arg=GIT_REPO={git_repo}'] diff --git a/buildtools/containerfiles/fedora:44.amd64 b/buildtools/containerfiles/fedora:44.amd64 index 736968b2..ecc1ed70 100644 --- a/buildtools/containerfiles/fedora:44.amd64 +++ b/buildtools/containerfiles/fedora:44.amd64 @@ -1,4 +1,4 @@ -FROM --platform=amd64 fedora:43 as build +FROM --platform=amd64 fedora:44 as build COPY install-build-pkgs build-mergerfs /tmp/ RUN /tmp/install-build-pkgs ARG BUILD_TIMESTAMP=0 diff --git a/buildtools/containerfiles/fedora:44.arm64 b/buildtools/containerfiles/fedora:44.arm64 index ca0bc04f..0ee220cc 100644 --- a/buildtools/containerfiles/fedora:44.arm64 +++ b/buildtools/containerfiles/fedora:44.arm64 @@ -1,4 +1,4 @@ -FROM --platform=arm64 fedora:43 as build +FROM --platform=arm64 fedora:44 as build COPY install-build-pkgs build-mergerfs /tmp/ RUN /tmp/install-build-pkgs ARG BUILD_TIMESTAMP=0 diff --git a/debian/rules b/debian/rules index e1d52bc2..8924dfdc 100755 --- a/debian/rules +++ b/debian/rules @@ -6,8 +6,11 @@ export DH_VERBOSE=1 %: dh $@ +override_dh_auto_build: + $(MAKE) release + override_dh_auto_install: $(MAKE) \ DESTDIR=$(CURDIR)/debian/mergerfs \ PREFIX=/usr \ - install + install \ No newline at end of file diff --git a/vendored/libfuse/Makefile b/vendored/libfuse/Makefile index d521f5db..d41928c8 100644 --- a/vendored/libfuse/Makefile +++ b/vendored/libfuse/Makefile @@ -32,9 +32,6 @@ UTILS := INSTALLUTILS := endif -ifeq ($(NDEBUG),1) -OPT_FLAGS := -O2 -DNDEBUG -else ifdef SANITIZE ifeq ($(SANITIZE),1) override SANITIZE := -fsanitize=address,undefined,leak @@ -42,15 +39,22 @@ else ifeq ($(filter -fsanitize=%,$(SANITIZE)),) override SANITIZE := -fsanitize=$(SANITIZE) endif endif -OPT_FLAGS := -O0 \ - -g \ - -fno-omit-frame-pointer \ - $(SANITIZE) \ - -fstack-protector-strong \ - -Wextra \ - -Werror \ - -Wno-unused-parameter \ - -DDEBUG + +COMMON_OPT_FLAGS += \ + -fstack-protector-strong \ + -Wextra \ + -Wno-unused-parameter + +DEBUG_OPT_FLAGS += \ + -O0 \ + -g3 \ + -fno-omit-frame-pointer \ + $(SANITIZE) + +ifdef RELEASE +OPT_FLAGS := -O3 $(COMMON_OPT_FLAGS) +else +OPT_FLAGS ?= $(DEBUG_OPT_FLAGS) $(COMMON_OPT_FLAGS) endif ifeq ($(LTO),1) @@ -151,10 +155,10 @@ $(BUILDDIR)/mount.mergerfs: $(BUILDDIR)/libfuse.a util/mount.mergerfs.cpp mount.mergerfs: $(BUILDDIR)/mount.mergerfs $(BUILDDIR)/.objs/%.c.o: lib/%.c | $(BUILDDIR)/stamp - $(CC) $(CFLAGS) $(FUSE_FLAGS) -c $< -o $@ + $(CC) $(CFLAGS) $(CPPFLAGS) $(FUSE_FLAGS) -c $< -o $@ $(BUILDDIR)/.objs/%.cpp.o: lib/%.cpp | $(BUILDDIR)/stamp - $(CXX) $(CXXFLAGS) $(FUSE_FLAGS) -c $< -o $@ + $(CXX) $(CXXFLAGS) $(CPPFLAGS) $(FUSE_FLAGS) -c $< -o $@ .PHONY: clean clean: @@ -176,6 +180,10 @@ install-utils: mergerfs-fusermount mount.mergerfs strip install: $(INSTALLUTILS) +.PHONY: release +release: + $(MAKE) RELEASE=1 NDEBUG=1 CPPFLAGS="$(CPPFLAGS) -DNDEBUG" all + .PHONY: objects strip utils install install-utils -include $(DEPS) From 12ac368e8b454dd5378b90d3094283641216e9ef Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Sat, 9 May 2026 19:20:20 -0500 Subject: [PATCH 02/23] Preserve CPPFLAGS in release builds - Append -DNDEBUG to existing CPPFLAGS in the release target. - Keep caller-provided build flags for Debian packaging. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 4dc0977a..07cfdf4d 100644 --- a/Makefile +++ b/Makefile @@ -369,7 +369,7 @@ endef .PHONY: release-amd64 release-arm64 release-armhf release-riscv64 .PHONY: release-sample release-static release-tarball release-installer release: - $(MAKE) RELEASE=1 CPPFLAGS=-DNDEBUG all + $(MAKE) RELEASE=1 CPPFLAGS="$(CPPFLAGS) -DNDEBUG" all release-all: $(call build_release,"all") release-sample: From dfe053a0ae21cf11ac20092269bd7de2e4f222f2 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 12:15:50 -0500 Subject: [PATCH 03/23] Require C++20-capable build toolchains Important changes: - Switch mergerfs and vendored libfuse Makefiles to -std=c++20. - Select the latest available GCC/G++ or toolset in container builds. - Verify compiler C++20 support during package installation. - Fix dynamic SysLog formatting for fmt C++20 checks. --- Makefile | 2 +- buildtools/build-mergerfs | 5 +- buildtools/containerfiles/static.installer | 5 +- buildtools/install-build-pkgs | 129 ++++++++++++++++++++- src/fuse_create.cpp | 2 +- src/mergerfs.cpp | 2 +- vendored/libfuse/Makefile | 2 +- vendored/libfuse/lib/debug.cpp | 4 +- 8 files changed, 140 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 07cfdf4d..9ef42bda 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ CXXFLAGS ?= \ -Wno-unused-result \ -pipe override CXXFLAGS += \ - -std=c++17 \ + -std=c++20 \ -MMD \ -MP override INC_FLAGS := \ diff --git a/buildtools/build-mergerfs b/buildtools/build-mergerfs index 75ac36c2..efb9140f 100755 --- a/buildtools/build-mergerfs +++ b/buildtools/build-mergerfs @@ -1,6 +1,10 @@ #!/bin/sh set -e +if [ -e /tmp/build-env ]; then + . /tmp/build-env +fi + if [ $# -ne 2 ]; then echo "usage: $0 " echo "argv: $@" @@ -30,7 +34,6 @@ if [ -e /usr/bin/apt-get ]; then elif [ -e /usr/bin/dnf ]; then make rpm elif [ -e /usr/bin/yum ]; then - . /opt/rh/devtoolset-9/enable make rpm elif [ -e /sbin/apk ]; then echo "NOT YET SUPPORTED" diff --git a/buildtools/containerfiles/static.installer b/buildtools/containerfiles/static.installer index ea0d2fee..0d3b29b8 100644 --- a/buildtools/containerfiles/static.installer +++ b/buildtools/containerfiles/static.installer @@ -1,6 +1,9 @@ ARG BUILD_TIMESTAMP=0 FROM --platform=linux/amd64 ubuntu:latest as build -RUN apt-get update && apt-get install -y makeself +COPY install-build-pkgs /tmp/ +RUN /tmp/install-build-pkgs && \ + DEBIAN_FRONTEND=noninteractive apt-get -qy --no-install-suggests --no-install-recommends install \ + makeself RUN mkdir -p /tmp/mergerfs/ RUN ls -lhd /pkgs RUN cp -v /pkgs/*static*.gz /tmp/mergerfs/ diff --git a/buildtools/install-build-pkgs b/buildtools/install-build-pkgs index 20eb3bba..7751df7b 100755 --- a/buildtools/install-build-pkgs +++ b/buildtools/install-build-pkgs @@ -1,39 +1,162 @@ #!/bin/sh +set -e + +BUILD_ENV=/tmp/build-env + +write_gcc_env() +{ + { + printf 'export CC=%s\n' "$1" + printf 'export CXX=%s\n' "$2" + } > "$BUILD_ENV" +} + +write_toolset_env() +{ + { + printf '. %s\n' "$1" + printf 'export CC=gcc\n' + printf 'export CXX=g++\n' + } > "$BUILD_ENV" +} + +latest_apt_gxx() +{ + apt-cache --names-only search '^g[+][+]-[0-9]' \ + | sed -n 's/^\(g++-[0-9][0-9]*\) .*/\1/p' \ + | sort -t- -k2,2V \ + | tail -n 1 +} + +latest_dnf_toolset() +{ + dnf -q list --showduplicates 'gcc-toolset-*-gcc-c++' 2>/dev/null \ + | sed -n 's/^gcc-toolset-\([0-9][0-9]*\)-gcc-c++\..*/\1/p' \ + | sort -V \ + | tail -n 1 +} + +latest_yum_toolset() +{ + yum -q list available 'devtoolset-*-gcc-c++' 2>/dev/null \ + | sed -n 's/^devtoolset-\([0-9][0-9]*\)-gcc-c++\..*/\1/p' \ + | sort -V \ + | tail -n 1 +} + +version_gt() +{ + highest="$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n 1)" + [ "$highest" = "$1" ] && [ "$1" != "$2" ] +} + +check_cxx20() +{ + cxx="${CXX:-g++}" + test_src=/tmp/mergerfs-cxx20-check.cpp + test_bin=/tmp/mergerfs-cxx20-check + + cat > "$test_src" <<'EOF' +template +concept Incrementable = requires(T value) { value + 1; }; + +static_assert(Incrementable); + +int main() +{ + return 0; +} +EOF + + if ! "$cxx" -std=c++20 "$test_src" -o "$test_bin" >/dev/null 2>&1; then + echo "error: $cxx does not support C++20" >&2 + exit 1 + fi + + rm -f "$test_src" "$test_bin" +} + +: > "$BUILD_ENV" + if [ -e /usr/bin/apt-get ]; then export DEBIAN_FRONTEND=noninteractive apt-get -qy update + cxx_pkg="$(latest_apt_gxx)" + if [ -n "$cxx_pkg" ]; then + gcc_pkg="gcc-${cxx_pkg#g++-}" + compiler_pkgs="$gcc_pkg $cxx_pkg" + write_gcc_env "$gcc_pkg" "$cxx_pkg" + else + compiler_pkgs="g++" + write_gcc_env gcc g++ + fi apt-get -qy --force-yes --no-install-suggests --no-install-recommends \ install \ ca-certificates \ build-essential \ dpkg-dev \ git \ - g++ \ + $compiler_pkgs \ debhelper \ lsb-release \ fakeroot + . "$BUILD_ENV" + check_cxx20 elif [ -e /usr/bin/dnf ]; then dnf -y update + toolset="$(latest_dnf_toolset)" dnf -y install \ git rpm-build gcc-c++ make which libatomic + default_gxx_major="$(g++ -dumpversion | cut -f1 -d.)" + if [ -n "$toolset" ] && version_gt "$toolset" "$default_gxx_major"; then + dnf -y install \ + "gcc-toolset-${toolset}-gcc" \ + "gcc-toolset-${toolset}-gcc-c++" + write_toolset_env "/opt/rh/gcc-toolset-${toolset}/enable" + else + write_gcc_env gcc g++ + fi + . "$BUILD_ENV" + check_cxx20 elif [ -e /usr/bin/yum ]; then yum -y update yum -y install \ git rpm-build gcc-c++ make which - yum -y install centos-release-scl - yum -y install devtoolset-9-gcc\* + yum -y install centos-release-scl || true + yum -y update + toolset="$(latest_yum_toolset)" + default_gxx_major="$(g++ -dumpversion | cut -f1 -d.)" + if [ -n "$toolset" ] && version_gt "$toolset" "$default_gxx_major"; then + yum -y install \ + "devtoolset-${toolset}-gcc" \ + "devtoolset-${toolset}-gcc-c++" + write_toolset_env "/opt/rh/devtoolset-${toolset}/enable" + else + write_gcc_env gcc g++ + fi + . "$BUILD_ENV" + check_cxx20 elif [ -e /usr/bin/zypper ]; then zypper update -y zypper install -y \ git rpm-build gcc-c++ make which + write_gcc_env gcc g++ + . "$BUILD_ENV" + check_cxx20 elif [ -e /sbin/apk ]; then apk add \ abuild git gcc g++ make \ linux-headers + write_gcc_env gcc g++ + . "$BUILD_ENV" + check_cxx20 elif [ -e /usr/sbin/pkg ]; then pkg install \ git gmake gcc elif [ -e /usr/bin/pacman ]; then pacman -Sy --noconfirm base-devel git fakeroot + write_gcc_env gcc g++ + . "$BUILD_ENV" + check_cxx20 fi diff --git a/src/fuse_create.cpp b/src/fuse_create.cpp index 49f61b7b..2fe50b40 100644 --- a/src/fuse_create.cpp +++ b/src/fuse_create.cpp @@ -334,7 +334,7 @@ _create(const fuse_req_ctx_t *ctx_, delete fi; ffi_->fh = 0; - SysLog::crit(msg); + SysLog::crit("{}",msg); fmt::println(stderr,"{}",msg); return -EIO; diff --git a/src/mergerfs.cpp b/src/mergerfs.cpp index a928eba6..c4ad8903 100644 --- a/src/mergerfs.cpp +++ b/src/mergerfs.cpp @@ -301,7 +301,7 @@ _warn_if_not_root() "mergerfs is not running as root" " and may not work correctly"; fmt::println(stderr,"* WARNING: {}",s); - SysLog::warning(s); + SysLog::warning("{}",s); } int diff --git a/vendored/libfuse/Makefile b/vendored/libfuse/Makefile index d41928c8..c22eaa4a 100644 --- a/vendored/libfuse/Makefile +++ b/vendored/libfuse/Makefile @@ -114,7 +114,7 @@ CXXFLAGS ?= \ -Wall \ -pipe override CXXFLAGS += \ - -std=c++17 \ + -std=c++20 \ -MMD \ -MP FUSERMOUNT_DIR := $(BINDIR) diff --git a/vendored/libfuse/lib/debug.cpp b/vendored/libfuse/lib/debug.cpp index d435a0d2..aaeb4f10 100644 --- a/vendored/libfuse/lib/debug.cpp +++ b/vendored/libfuse/lib/debug.cpp @@ -1191,7 +1191,7 @@ fuse_syslog_fuse_init_in(const struct fuse_init_in *arg_) output.pop_back(); output += ");"; - SysLog::info(output); + SysLog::info("{}",output); } void @@ -1239,7 +1239,7 @@ fuse_syslog_fuse_init_out(const struct fuse_init_out *arg_) output.pop_back(); output += ");"; - SysLog::info(output); + SysLog::info("{}",output); } From 56285fc6e5c81ef0ba1de76b7da98201216f0aed Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 13:56:09 -0500 Subject: [PATCH 04/23] Pass annobin override to RPM toolset builds Important changes: - Add Makefile RPMBUILD and RPMBUILD_FLAGS overrides for the rpm target. - Detect /opt/rh GCC toolsets before RPM packaging. - Pass --undefine _annotated_build to avoid Rocky 8 annobin plugin failures. --- Makefile | 4 +++- buildtools/build-mergerfs | 13 +++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9ef42bda..1d785a24 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ MKDIR ?= mkdir MKTEMP ?= mktemp MV ?= mv RM ?= rm +RPMBUILD ?= rpmbuild +RPMBUILD_FLAGS ?= SED ?= sed STRIP ?= strip TAR ?= tar @@ -343,7 +345,7 @@ rpm: tarball $(SED) 's/__VERSION__/$(VERSION)/g' mergerfs.spec > \ rpmbuild/SOURCES/mergerfs.spec cp -ar mergerfs-$(VERSION).tar.gz rpmbuild/SOURCES - rpmbuild -ba rpmbuild/SOURCES/mergerfs.spec \ + $(RPMBUILD) $(RPMBUILD_FLAGS) -ba rpmbuild/SOURCES/mergerfs.spec \ --define "_topdir $(CURDIR)/rpmbuild" .PHONY: install-build-pkgs diff --git a/buildtools/build-mergerfs b/buildtools/build-mergerfs index efb9140f..baea10db 100755 --- a/buildtools/build-mergerfs +++ b/buildtools/build-mergerfs @@ -5,6 +5,15 @@ if [ -e /tmp/build-env ]; then . /tmp/build-env fi +rpmbuild_flags() +{ + case "$(command -v "${CXX:-g++}")" in + /opt/rh/*) + printf '%s' '--undefine _annotated_build' + ;; + esac +} + if [ $# -ne 2 ]; then echo "usage: $0 " echo "argv: $@" @@ -32,9 +41,9 @@ mkdir "/build" if [ -e /usr/bin/apt-get ]; then make deb elif [ -e /usr/bin/dnf ]; then - make rpm + make RPMBUILD_FLAGS="$(rpmbuild_flags)" rpm elif [ -e /usr/bin/yum ]; then - make rpm + make RPMBUILD_FLAGS="$(rpmbuild_flags)" rpm elif [ -e /sbin/apk ]; then echo "NOT YET SUPPORTED" exit 1 From 1e0bd5498911d3d1717c5614043fffe80bbcac9a Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 07:47:44 -0500 Subject: [PATCH 05/23] Move statvfs cache to use shared mutex Its read heavy and micro benchmarks show no regression on the low end and 30%+ throughput improvements on the highend. --- src/fs_statvfs_cache.cpp | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/fs_statvfs_cache.cpp b/src/fs_statvfs_cache.cpp index 1d2fa708..891708da 100644 --- a/src/fs_statvfs_cache.cpp +++ b/src/fs_statvfs_cache.cpp @@ -21,8 +21,8 @@ #include "fs_statvfs.hpp" #include "statvfs_util.hpp" -#include "mutex.hpp" - +#include +#include #include #include @@ -38,9 +38,11 @@ struct Element typedef std::unordered_map statvfs_cache; -static u64 g_timeout = 0; -static statvfs_cache g_cache; -static Mutex g_cache_lock; +// Cache is read heavy. Use a shared mutex so concurrent reads can +// work in parallel and only inserts or refreshes take a unique lock. +static std::shared_mutex g_cache_mutex; +static u64 g_timeout = 0; +static statvfs_cache g_cache; static @@ -70,29 +72,35 @@ int fs::statvfs_cache(const std::string &path_, struct statvfs *st_) { - int rv; - Element *e; u64 now; if(g_timeout == 0) return fs::statvfs(path_,st_); - rv = 0; now = ::_get_time(); - mutex_lock(g_cache_lock); + { + std::shared_lock lk(g_cache_mutex); - e = &g_cache[path_]; + auto it = g_cache.find(path_); + if((it != g_cache.end()) && ((now - it->second.time) <= g_timeout)) + { + *st_ = it->second.st; + return 0; + } + } - if((now - e->time) > g_timeout) + int rv = 0; + std::unique_lock lk(g_cache_mutex); + + Element &e = g_cache[path_]; + if((now - e.time) > g_timeout) { - e->time = now; - rv = fs::statvfs(path_,&e->st); + e.time = now; + rv = fs::statvfs(path_,&e.st); } - *st_ = e->st; - - mutex_unlock(g_cache_lock); + *st_ = e.st; return rv; } From 8608b2be4646503023cca76f614ce60554fcb0a5 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 07:59:07 -0500 Subject: [PATCH 06/23] Clarify releasedir readdir synchronization barrier - Document the lock/unlock pair as an acquire/release barrier. - Explain that releasedir waits for in-flight readdir calls before destroy/free. - Warn against removing the synchronization despite it not being a critical section. --- vendored/libfuse/lib/fuse.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/vendored/libfuse/lib/fuse.cpp b/vendored/libfuse/lib/fuse.cpp index 92358e48..38a480df 100644 --- a/vendored/libfuse/lib/fuse.cpp +++ b/vendored/libfuse/lib/fuse.cpp @@ -2543,7 +2543,11 @@ fuse_lib_releasedir(fuse_req_t *req_, f.ops.releasedir(&req_->ctx, &ffi); - /* Done to keep race condition between last readdir reply and the unlock */ + // Acquire/release barrier: not a real critical section. A + // concurrent readdir may still hold dh->lock at this point. This + // lock+unlock pair blocks until the last readdir drops the lock + // guaranteeing that the subsequent destroy/free sees no in-flight + // readdir accessing dh->d. Do NOT remove. mutex_lock(dh->lock); mutex_unlock(dh->lock); mutex_destroy(dh->lock); From 8b17d5f35e544511123f81d3ece6743d34258e5f Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 08:24:03 -0500 Subject: [PATCH 07/23] Synchronize node updates with path release - Add helpers that release path locks and update node state under f.lock. - Update getattr and setattr stat-cache invalidation while preserving ESTALE handling. - Rework create, open, tmpfile, and set_path_info to mutate lookup/open state under the shared lock. --- vendored/libfuse/lib/fuse.cpp | 288 +++++++++++++++++++++++----------- 1 file changed, 197 insertions(+), 91 deletions(-) diff --git a/vendored/libfuse/lib/fuse.cpp b/vendored/libfuse/lib/fuse.cpp index 38a480df..ea875882 100644 --- a/vendored/libfuse/lib/fuse.cpp +++ b/vendored/libfuse/lib/fuse.cpp @@ -1064,6 +1064,29 @@ free_path_wrlock(uint64_t nodeid, free(path); } +static +void +update_stat(node_t *node_, + const struct stat *stnew_) +{ + uint32_t crc32b; + + // This function can only invalidate an existing fingerprint; it + // never arms a new one. Arming (writing a non-zero CRC into + // state_crc32b) happens exclusively in open_auto_cache(), which is + // the single point that decides a file's page cache is worth + // retaining across opens. If stat_crc32b is 0 here the cache is + // already considered invalid and we have nothing to compare + // against. + if(node_->stat_crc32b == 0) + return; + + crc32b = stat_crc32b(stnew_); + + if(crc32b != node_->stat_crc32b) + node_->stat_crc32b = 0; +} + static void free_path(uint64_t nodeid, @@ -1073,6 +1096,59 @@ free_path(uint64_t nodeid, free_path_wrlock(nodeid,NULL,path); } +static +node_t* +free_path_and_inc_open(u64 free_nodeid_, + char *path_, + u64 inc_open_ino_, + bool remember_too_) +{ + node_t *node; + + mutex_lock(f.lock); + + unlock_path(free_nodeid_,NULL,NULL); + node = get_node(inc_open_ino_); + if(node) + { + if(remember_too_ && remember_nodes()) + remember_node(node); + node->open_count++; + } + if(f.lockq) + wake_up_queued(); + + mutex_unlock(f.lock); + free(path_); + + return node; +} + +static +bool +free_path_and_update_stat(u64 nodeid_, + char *path_, + const struct stat *stnew_) +{ + bool ok; + node_t *node; + + mutex_lock(f.lock); + + unlock_path(nodeid_,NULL,NULL); + node = get_node(nodeid_); + ok = (node != NULL); + if(ok) + update_stat(node,stnew_); + if(f.lockq) + wake_up_queued(); + + mutex_unlock(f.lock); + free(path_); + + return ok; +} + static void free_path2(uint64_t nodeid1, @@ -1208,52 +1284,65 @@ node_open(const node_t *node_) return ((node_ != NULL) && (node_->open_count > 0)); } -static -void -update_stat(node_t *node_, - const struct stat *stnew_) -{ - uint32_t crc32b; - - // This function can only invalidate an existing fingerprint; it - // never arms a new one. Arming (writing a non-zero CRC into - // state_crc32b) happens exclusively in open_auto_cache(), which is - // the single point that decides a file's page cache is worth - // retaining across opens. If stat_crc32b is 0 here the cache is - // already considered invalid and we have nothing to compare - // against. - if(node_->stat_crc32b == 0) - return; - - crc32b = stat_crc32b(stnew_); - - if(crc32b != node_->stat_crc32b) - node_->stat_crc32b = 0; -} - static int -set_path_info(uint64_t nodeid, - const char *name, - struct fuse_entry_param *e, - bool remember = true) +set_path_info(uint64_t nodeid_, + const char *name_, + struct fuse_entry_param *e_, + bool remember_ = true) { node_t *node; - node = find_node(nodeid,name,remember); - if(node == NULL) - return -ENOMEM; - - e->ino = node->nodeid; - e->generation = ((e->ino == FUSE_ROOT_ID) ? 0 : f.nodeid_gen.generation); - mutex_lock(f.lock); - update_stat(node,&e->attr); + + if(!name_) + { + node = get_node(nodeid_); + if(node == NULL) + goto out_unlock; + } + else + { + node = lookup_node(nodeid_,name_); + } + + if(node == NULL) + { + node = node_alloc(); + if(node == NULL) + goto out_unlock; + + node->nodeid = generate_nodeid(&f.nodeid_gen); + if(remember_ && remember_nodes()) + remember_node(node); + + if(hash_name(node,nodeid_,name_) == -1) + { + free_node(node); + node = NULL; + goto out_unlock; + } + + hash_id(node); + } + + inc_nlookup(node); + + e_->ino = node->nodeid; + e_->generation = ((e_->ino == FUSE_ROOT_ID) ? 0 : f.nodeid_gen.generation); + + update_stat(node,&e_->attr); + mutex_unlock(f.lock); - set_stat(e->ino,&e->attr); + set_stat(e_->ino,&e_->attr); return 0; + + out_unlock: + mutex_unlock(f.lock); + + return -ENOMEM; } /* @@ -1474,26 +1563,37 @@ fuse_lib_getattr(fuse_req_t *req_, if(!err) { + bool node_ok = true; if(fusepath != NULL) - err = f.ops.getattr(&req_->ctx,&fusepath[1],&buf,&timeouts); + { + err = f.ops.getattr(&req_->ctx,&fusepath[1],&buf,&timeouts); + if(!err) + node_ok = free_path_and_update_stat(hdr_->nodeid,fusepath,&buf); + else + free_path(hdr_->nodeid,fusepath); + } else - err = f.ops.fgetattr(&req_->ctx,fh,&buf,&timeouts); + { + err = f.ops.fgetattr(&req_->ctx,fh,&buf,&timeouts); + if(!err) + { + mutex_lockguard(f.lock); + node = get_node(hdr_->nodeid); + node_ok = (node != NULL); + if(node_ok) + update_stat(node,&buf); + } + } - free_path(hdr_->nodeid,fusepath); + if(!err && !node_ok) + { + fuse_reply_err(req_,ESTALE); + return; + } } if(!err) { - { - mutex_lockguard(f.lock); - node = get_node(hdr_->nodeid); - if(node == NULL) - { - fuse_reply_err(req_,ESTALE); - return; - } - update_stat(node,&buf); - } set_stat(hdr_->nodeid,&buf); fuse_reply_attr(req_,&buf,timeouts.attr); } @@ -1663,29 +1763,51 @@ fuse_lib_setattr(fuse_req_t *req_, f.ops.getattr(&req_->ctx,&fusepath[1],&stbuf,&timeouts) : f.ops.fgetattr(&req_->ctx,fh,&stbuf,&timeouts)); - free_path(hdr_->nodeid,fusepath); + bool stat_updated = false; + if(!err && (fusepath != NULL)) + { + node_t *node; + mutex_lock(f.lock); + unlock_path(hdr_->nodeid,NULL,NULL); + node = get_node(hdr_->nodeid); + if(node) + update_stat(node,&stbuf); + if(f.lockq) + wake_up_queued(); + mutex_unlock(f.lock); + free(fusepath); + stat_updated = true; + } + else + { + free_path(hdr_->nodeid,fusepath); + } + + if(!err) + { + if(!stat_updated) + { + node_t *node; + mutex_lockguard(f.lock); + node = get_node(hdr_->nodeid); + if(node) + update_stat(node,&stbuf); + } + + set_stat(hdr_->nodeid,&stbuf); + fuse_reply_attr(req_,&stbuf,timeouts.attr); + } + else + { + fuse_reply_err(req_,err); + } + return; } - if(!err) - { - { - node_t *node; - mutex_lockguard(f.lock); - - node = get_node(hdr_->nodeid); - if(node) - update_stat(node,&stbuf); - } - - set_stat(hdr_->nodeid,&stbuf); - fuse_reply_attr(req_,&stbuf,timeouts.attr); - } - else - { - fuse_reply_err(req_,err); - } + fuse_reply_err(req_,err); } + static void fuse_lib_access(fuse_req_t *req_, @@ -2089,15 +2211,8 @@ fuse_lib_create(fuse_req_t *req_, if(!err) { - mutex_lock(f.lock); - node_t *node = get_node(e.ino); - if(node) - { - if(remember_nodes()) - remember_node(node); - node->open_count++; - } - mutex_unlock(f.lock); + free_path_and_inc_open(hdr_->nodeid,fusepath,e.ino,true); + fusepath = NULL; if(fuse_reply_create(req_,&e,&ffi) == -ENOENT) { @@ -2196,11 +2311,9 @@ fuse_lib_open(fuse_req_t *req_, if(!err) { - mutex_lock(f.lock); - node_t *node = get_node(hdr_->nodeid); - if(node) - node->open_count++; - mutex_unlock(f.lock); + free_path_and_inc_open(hdr_->nodeid,fusepath,hdr_->nodeid,false); + fusepath = NULL; + /* The open syscall was interrupted,so it must be cancelled */ if(fuse_reply_open(req_,&ffi) == -ENOENT) fuse_do_release(&req_->ctx,hdr_->nodeid,&ffi); @@ -2928,15 +3041,8 @@ fuse_lib_tmpfile(fuse_req_t *req_, if(!err) { - mutex_lock(f.lock); - node_t *node = get_node(e.ino); - if(node) - { - if(remember_nodes()) - remember_node(node); - node->open_count++; - } - mutex_unlock(f.lock); + free_path_and_inc_open(hdr_->nodeid,fusepath,e.ino,true); + fusepath = NULL; if(fuse_reply_create(req_,&e,&ffi) == -ENOENT) { From 231aac15cd7298a0acb71b406c215a8009e8af1c Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 09:17:48 -0500 Subject: [PATCH 08/23] Use shared write locks until ENOSPC relocation - Replace the per-file Mutex with std::shared_mutex for shared write access. - Run direct and cached writes under a shared lock while the fd is stable. - Retry failed writes under a unique lock before move-on-ENOSPC relocation. --- src/fileinfo.hpp | 15 ++++++-- src/fuse_write.cpp | 96 ++++++++++++++++++++++++---------------------- 2 files changed, 62 insertions(+), 49 deletions(-) diff --git a/src/fileinfo.hpp b/src/fileinfo.hpp index 6a2866f7..f3524205 100644 --- a/src/fileinfo.hpp +++ b/src/fileinfo.hpp @@ -48,10 +48,10 @@ #include "fh.hpp" #include "fs_path.hpp" -#include "mutex.hpp" - #include "base_types.h" +#include + class FileInfo : public FH { @@ -97,7 +97,16 @@ public: int fd; Branch branch; u32 direct_io:1; - Mutex mutex; + // Serializes the fd state across concurrent writes on the same open + // file. Concurrent writes happen with: + // 1) writeback-cache + page-cache mode + // 2) parallel_direct_writes + direct_io=true + // + // Common case (plain pwrite on stable fd) only needs a shared lock + // since the kernel already serializes fd mutations. The danger is + // the fs::dup2() rebinding when moveonenospc activates. Writers + // take shared lock; move takes unique. + std::shared_mutex mutex; }; inline diff --git a/src/fuse_write.cpp b/src/fuse_write.cpp index 3bdcb0e0..b91c7ba0 100644 --- a/src/fuse_write.cpp +++ b/src/fuse_write.cpp @@ -33,6 +33,9 @@ #include "fuse.h" +#include +#include + static bool @@ -117,47 +120,11 @@ _move_and_pwriten(char const *buf_, // N bytes written (short writes included) // -errno on error // See libfuse/include/fuse.h for more details -static -int -_write_direct_io(const char *buf_, - const size_t count_, - const off_t offset_, - FileInfo *fi_) -{ - ssize_t rv; - - rv = fs::pwrite(fi_->fd,buf_,count_,offset_); - if(::_out_of_space(rv)) - rv = ::_move_and_pwrite(buf_,count_,offset_,fi_,rv); - - return rv; -} - // When not in direct_io mode write's return value is more complex. // 0 or less than `count` on EOF // `count` on non-errors // -errno on error // See libfuse/include/fuse.h for more details -static -int -_write_cached(const char *buf_, - const size_t count_, - const off_t offset_, - FileInfo *fi_) -{ - int err; - ssize_t rv; - - rv = fs::pwriten(fi_->fd,buf_,count_,offset_,&err); - if(err == 0) - return rv; - if(err && !::_out_of_space(err)) - return err; - - rv = ::_move_and_pwriten(buf_,count_,offset_,fi_,err,rv); - - return rv; -} static int @@ -175,19 +142,56 @@ _write(const fuse_req_ctx_t *ctx_, // Concurrent writes can only happen if: // 1) writeback-cache is enabled and using page caching - // 2) parallel_direct_writes is enabled and file has `direct_io=true` - // Will look into selectively locking in the future - // A reader/writer lock would probably be the best option given - // the expense of the write itself in comparison. Alternatively, - // could change the move file behavior to use a known target file - // and have threads use O_EXCL and back off and wait for the - // transfer to complete before retrying. - mutex_lockguard(fi->mutex); + // 2) parallel_direct_writes is enabled and file has + // `direct_io=true` + + ssize_t rv; if(fi->direct_io) - return ::_write_direct_io(buf_,count_,offset_,fi); + { + { + std::shared_lock slk(fi->mutex); + rv = fs::pwrite(fi->fd,buf_,count_,offset_); + } - return ::_write_cached(buf_,count_,offset_,fi); + if(not ::_out_of_space(rv)) + return rv; + + std::unique_lock ulk(fi->mutex); + // Re-check under exclusive lock: another writer may have + // already moved the file and retired. If so the pwrite should + // now succeed without our own move call. + rv = fs::pwrite(fi->fd,buf_,count_,offset_); + if(::_out_of_space(rv)) + rv = ::_move_and_pwrite(buf_,count_,offset_,fi,rv); + return rv; + } + else + { + int err; + ssize_t written; + + { + std::shared_lock slk(fi->mutex); + written = fs::pwriten(fi->fd,buf_,count_,offset_,&err); + } + + if(err == 0) + return written; + if(err && not ::_out_of_space(err)) + return err; + + std::unique_lock ulk(fi->mutex); + // Re-check under exclusive lock as another move may have + // already run. + written = fs::pwriten(fi->fd,buf_,count_,offset_,&err); + if(err == 0) + return written; + if(err && not ::_out_of_space(err)) + return err; + + return ::_move_and_pwriten(buf_,count_,offset_,fi,err,written); + } } int From b13727abde41d9bfb20215f7aa494533ee73bb49 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 10:58:01 -0500 Subject: [PATCH 09/23] Document readdir merge mutex scope - Explain that the mutex is per readdir call. - Document that only dirent deduplication and list append are serialized. - Clarify that per-branch opendir and getdents64 work runs outside the critical section. --- src/fuse_readdir_cor.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/fuse_readdir_cor.cpp b/src/fuse_readdir_cor.cpp index 596a27f4..a8ef5c21 100644 --- a/src/fuse_readdir_cor.cpp +++ b/src/fuse_readdir_cor.cpp @@ -43,6 +43,12 @@ FUSE::ReadDirCOR::~ReadDirCOR() #include "fuse_readdir_cor_readdir.icpp" #endif +// Uses a per-call mutex. Lifetime is one readdir operation. It +// serializes the dirent-merge step (dedup'ing of names + append to +// dirents list) across at-most-branches-worth of parallel worker +// tasks. Per-branch opendir + getdents64 run fully outside the +// critical section, so the lock holds only long enough to merge each +// batch. static inline int From 608cd007646a69092ef6432199abf56c04aa7f17 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 11:30:39 -0500 Subject: [PATCH 10/23] Use atomic open file reference counts - Use relaxed increment under the map bucket lock when duplicating file handles. - Use acq_rel decrement when releasing and checking for final references. - Load moved ref_count with relaxed ordering during map insertion. --- src/fuse_open.cpp | 11 ++++++++++- src/fuse_release.cpp | 14 ++++++++++++-- src/state.hpp | 6 +++++- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/src/fuse_open.cpp b/src/fuse_open.cpp index 6bdc3e70..1d247079 100644 --- a/src/fuse_open.cpp +++ b/src/fuse_open.cpp @@ -367,10 +367,19 @@ _open(const fuse_req_ctx_t *ctx_, // The ref increment keeps fuse_release.cpp from erasing and // freeing the canonical entry while we duplicate its fd. + // + // memory_order_relaxed is sufficient for the increment. The + // concurrent_flat_map bucket lock held inside visit() already + // provides the happens-before relationship with other mutators + // of the same key (the release() erase_if() callback and other + // open() visitors.) The lock's release/acquire semantics + // publish the ref_count update to subsequent observers of the + // same entry, so we only need atomicity from the fetch_add + // itself. of.visit(ctx_->nodeid, [&](auto &v_) { - v_.second.ref_count++; + v_.second.ref_count.fetch_add(1,std::memory_order_relaxed); fi = v_.second.fi; backing_id = v_.second.backing_id; }); diff --git a/src/fuse_release.cpp b/src/fuse_release.cpp index 17bbb0cd..e8ddb1e6 100644 --- a/src/fuse_release.cpp +++ b/src/fuse_release.cpp @@ -81,11 +81,21 @@ FUSE::release(cu64 nodeid_, FileInfo *canonical_fi = nullptr; FileInfo *fh_fi_to_free = nullptr; + // memory_order_acq_rel on the decrement gives us the canonical + // refcount-publication pattern: + // - release: prior writes to the OpenFile by this thread are + // visible to whichever thread observes the final 0. + // - acquire: the thread that drops the last ref acquires every + // prior ref holder's writes before tearing down the entry. + // The concurrent_flat_map bucket lock already serializes concurrent + // visitors of this key so the acq_rel is good for any observer that + // might examine ref_count outside a visit(). of.erase_if(nodeid_, [&](auto &v_) { - v_.second.ref_count--; - if(v_.second.ref_count > 0) + const int prev = + v_.second.ref_count.fetch_sub(1,std::memory_order_acq_rel); + if(prev > 1) { if(fi_ != v_.second.fi) fh_fi_to_free = fi_; diff --git a/src/state.hpp b/src/state.hpp index f5e5867c..cabcbbd5 100644 --- a/src/state.hpp +++ b/src/state.hpp @@ -103,8 +103,12 @@ public: { } + // Move constructor runs during initial insertion into the + // concurrent_flat_map before the new entry is reachable by the + // other threads. There is no concurrent observer of the moved + // value at this point so relaxed load is sufficient. OpenFile(OpenFile &&o_) noexcept - : ref_count(o_.ref_count.load()), + : ref_count(o_.ref_count.load(std::memory_order_relaxed)), backing_id(o_.backing_id), fi(o_.fi) { From 8734847968edf60f65cce7021347f67cbddaa4d4 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 16:27:55 -0500 Subject: [PATCH 11/23] Guard branch snapshots with shared_mutex Important changes: - Replace atomic shared_ptr operations with shared_mutex-protected snapshots. - Retry copy-on-write branch updates when another writer publishes first. - Keep long-running parsing and read-only filesystem checks outside branch locks. --- src/branches.cpp | 96 ++++++++++++++++++++++++++++++++++-------------- src/branches.hpp | 18 +++++++-- 2 files changed, 82 insertions(+), 32 deletions(-) diff --git a/src/branches.cpp b/src/branches.cpp index 4ab59707..bc942be0 100644 --- a/src/branches.cpp +++ b/src/branches.cpp @@ -28,8 +28,10 @@ #include "str.hpp" #include "syslog.hpp" -#include +#include #include +#include +#include #include @@ -437,24 +439,44 @@ Branches::from_string(const std::string_view str_) Branches::Ptr impl; Branches::Ptr new_impl; - impl = std::atomic_load(&_impl); - new_impl = std::make_shared(&minfreespace); - *new_impl = *impl; - rv = new_impl->from_string(str_); - if(rv < 0) - return rv; + for(;;) + { + { + std::shared_lock lk(_mutex); + impl = _impl; + } - std::atomic_store(&_impl,new_impl); + *new_impl = *impl; - return 0; + rv = new_impl->from_string(str_); + if(rv < 0) + return rv; + + { + std::unique_lock lk(_mutex); + if(impl != _impl) + continue; + + _impl = std::move(new_impl); + } + + return 0; + } } std::string Branches::to_string(void) const { - return std::atomic_load(&_impl)->to_string(); + Branches::Ptr impl; + + { + std::shared_lock lk(_mutex); + impl = _impl; + } + + return impl->to_string(); } void @@ -462,32 +484,50 @@ Branches::find_and_set_mode_ro() { Branches::Ptr impl; Branches::Ptr new_impl; + std::vector ro_paths; - impl = std::atomic_load(&_impl); new_impl = std::make_shared(&minfreespace); - *new_impl = *impl; - - bool changed = false; - for(auto &branch : *new_impl) + for(;;) { - if(branch.mode != Branch::Mode::RW) - continue; + ro_paths.clear(); - if(!fs::is_rofs_but_not_mounted_ro(branch.path)) - continue; + { + std::shared_lock lk(_mutex); + impl = _impl; + } - SysLog::warning("branch `{}` found to be readonly - setting its mode=RO", - branch.path.string()); + *new_impl = *impl; - branch.mode = Branch::Mode::RO; - changed = true; + for(auto &branch : *new_impl) + { + if(branch.mode != Branch::Mode::RW) + continue; + + if(!fs::is_rofs_but_not_mounted_ro(branch.path)) + continue; + + branch.mode = Branch::Mode::RO; + ro_paths.emplace_back(branch.path); + } + + if(ro_paths.empty()) + return; + + { + std::unique_lock lk(_mutex); + if(impl != _impl) + continue; + + _impl = std::move(new_impl); + } + + for(const auto &path : ro_paths) + SysLog::warning("branch `{}` found to be readonly - setting its mode=RO", + path.string()); + + return; } - - if(not changed) - return; - - std::atomic_store(&_impl,new_impl); } SrcMounts::SrcMounts(Branches &b_) diff --git a/src/branches.hpp b/src/branches.hpp index 54f85959..dae01822 100644 --- a/src/branches.hpp +++ b/src/branches.hpp @@ -23,9 +23,9 @@ #include "strvec.hpp" #include "tofrom_string.hpp" -#include #include #include +#include #include #include @@ -69,7 +69,8 @@ public: u64 minfreespace = MINFREESPACE_DEFAULT; private: - mutable std::shared_ptr _impl; + Branches::Ptr _impl; + mutable std::shared_mutex _mutex; public: Branches() @@ -82,8 +83,17 @@ public: std::string to_string(void) const final; public: - operator Ptr() const { return std::atomic_load(&_impl); } - Ptr operator->() const { return std::atomic_load(&_impl); } + operator Ptr() const + { + std::shared_lock lk(_mutex); + return _impl; + } + + Ptr operator->() const + { + std::shared_lock lk(_mutex); + return _impl; + } public: void find_and_set_mode_ro(); From 02dc36080cdfa1b7bac18543c6bc789b43f8f1d6 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 17:41:36 -0500 Subject: [PATCH 12/23] Use shared_mutex for ReadDir state - Replace atomic shared pointer loads/stores with shared/unique locking. - Store the configured readdir mode directly as std::string. - Copy the active ReadDir implementation under lock before invoking it. --- src/fuse_readdir.cpp | 36 +++++++++++++++++++++--------------- src/fuse_readdir.hpp | 7 ++++--- 2 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/fuse_readdir.cpp b/src/fuse_readdir.cpp index d506d3d8..f26c34f1 100644 --- a/src/fuse_readdir.cpp +++ b/src/fuse_readdir.cpp @@ -48,30 +48,29 @@ FUSE::ReadDir::ReadDir(const std::string_view s_) std::string FUSE::ReadDir::to_string() const { - std::shared_ptr str; + std::shared_lock lk(_mutex); - str = std::atomic_load(&_str); - - return (*str); + return _str; } int FUSE::ReadDir::from_string(const std::string_view str_) { - std::shared_ptr tmp_str; std::shared_ptr tmp_readdir; tmp_readdir = FUSE::ReadDirFactory::make(str_); if(!tmp_readdir) return -EINVAL; - tmp_str = std::make_shared(str_); - std::atomic_store(&_str,tmp_str); + { + std::unique_lock lk(_mutex); - if(_initialized == false) - return 0; + _str = str_; + if(_initialized == false) + return 0; - std::atomic_store(&_impl,tmp_readdir); + _impl = std::move(tmp_readdir); + } return 0; } @@ -114,7 +113,10 @@ FUSE::ReadDir::operator()(const fuse_req_ctx_t *ctx_, int rv; std::shared_ptr readdir; - readdir = std::atomic_load(&_impl); + { + std::shared_lock lk(_mutex); + readdir = _impl; + } if(!readdir) fatal::abort("readdir impl is null"); @@ -128,10 +130,14 @@ FUSE::ReadDir::operator()(const fuse_req_ctx_t *ctx_, void FUSE::ReadDir::initialize() { - std::shared_ptr str; + std::string str; - str = std::atomic_load(&_str); - _initialized = true; + { + std::unique_lock lk(_mutex); - from_string(*str); + str = _str; + _initialized = true; + } + + from_string(str); } diff --git a/src/fuse_readdir.hpp b/src/fuse_readdir.hpp index afcff6da..15dd0b8c 100644 --- a/src/fuse_readdir.hpp +++ b/src/fuse_readdir.hpp @@ -22,9 +22,9 @@ #include "tofrom_string.hpp" #include "fuse_readdir_base.hpp" -#include #include #include +#include namespace FUSE @@ -59,8 +59,9 @@ namespace FUSE void initialize(); private: - std::atomic _initialized = false; - std::shared_ptr _str; + bool _initialized = false; + mutable std::shared_mutex _mutex; + std::string _str; std::shared_ptr _impl; }; } From 30d59d4458ba347e9a3ad198e8fd2f611490ce11 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 18:30:40 -0500 Subject: [PATCH 13/23] Use shared_mutex for fuse log config - Guard fuse_cfg_t log file and filepath shared pointers with a shared mutex - Replace atomic shared pointer loads and stores with shared and unique locks --- vendored/libfuse/include/fuse_cfg.hpp | 2 ++ vendored/libfuse/lib/fuse_cfg.cpp | 20 +++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/vendored/libfuse/include/fuse_cfg.hpp b/vendored/libfuse/include/fuse_cfg.hpp index cdd9d00e..72f3edfb 100644 --- a/vendored/libfuse/include/fuse_cfg.hpp +++ b/vendored/libfuse/include/fuse_cfg.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #define FUSE_CFG_INVALID_ID -1 @@ -33,6 +34,7 @@ struct fuse_cfg_t std::shared_ptr log_filepath() const; void log_filepath(const std::string &); private: + mutable std::shared_mutex _log_mutex; std::shared_ptr _log_file = sp(stderr,[](FILE*){}); std::shared_ptr _log_filepath = sp(); diff --git a/vendored/libfuse/lib/fuse_cfg.cpp b/vendored/libfuse/lib/fuse_cfg.cpp index 574793f7..eb1e9e1b 100644 --- a/vendored/libfuse/lib/fuse_cfg.cpp +++ b/vendored/libfuse/lib/fuse_cfg.cpp @@ -1,5 +1,7 @@ #include "fuse_cfg.hpp" +#include + fuse_cfg_t fuse_cfg; bool @@ -23,24 +25,32 @@ fuse_cfg_t::valid_umask() const std::shared_ptr fuse_cfg_t::log_file() const { - return std::atomic_load(&_log_file); + std::shared_lock lk(_log_mutex); + + return _log_file; } void fuse_cfg_t::log_file(std::shared_ptr f_) { - std::atomic_store(&_log_file, f_); + std::unique_lock lk(_log_mutex); + + _log_file = std::move(f_); } std::shared_ptr fuse_cfg_t::log_filepath() const { - return std::atomic_load(&_log_filepath); + std::shared_lock lk(_log_mutex); + + return _log_filepath; } void fuse_cfg_t::log_filepath(const std::string &s_) { - std::atomic_store(&_log_filepath, - std::make_shared(s_)); + auto filepath = std::make_shared(s_); + std::unique_lock lk(_log_mutex); + + _log_filepath = std::move(filepath); } From 5f7fa17ad82d4054513e3eae80ac3ef129960bba Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Wed, 13 May 2026 19:26:37 -0500 Subject: [PATCH 14/23] Inline ioprio disabled fast path - Expose the ioprio enabled flag as an atomic for relaxed inline checks - Inline SetFrom construction so disabled read/write paths avoid a function call - Move priority application into a slow helper used only when ioprio is enabled --- src/ioprio.cpp | 20 ++++++++------------ src/ioprio.hpp | 27 +++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/src/ioprio.cpp b/src/ioprio.cpp index 0e3abe13..3ed1ba1a 100644 --- a/src/ioprio.cpp +++ b/src/ioprio.cpp @@ -32,8 +32,12 @@ enum IOPRIO_WHO_PROCESS = 1 }; +namespace ioprio +{ + std::atomic _enabled{false}; +} + thread_local int ioprio::SetFrom::thread_prio = -1; -bool _enabled = false; int ioprio::get(const int who_) @@ -69,22 +73,14 @@ ioprio::set(const int who_, void ioprio::enable(const bool enable_) { - _enabled = enable_; + _enabled.store(enable_,std::memory_order_relaxed); } -bool -ioprio::enabled() -{ - return _enabled; -} - -ioprio::SetFrom::SetFrom(const pid_t pid_) +void +ioprio::SetFrom::_slow_apply(const pid_t pid_) { int client_prio; - if(!_enabled) - return; - client_prio = ioprio::get(pid_); if(client_prio < 0) return; diff --git a/src/ioprio.hpp b/src/ioprio.hpp index acdc455d..1f272a4f 100644 --- a/src/ioprio.hpp +++ b/src/ioprio.hpp @@ -18,13 +18,27 @@ #pragma once +#include + #include namespace ioprio { + // Exposed so the disabled-path check in SetFrom can inline into + // every fuse_read/fuse_write as an atomic load + branch, with no + // function overhead. + extern std::atomic _enabled; + + [[gnu::always_inline]] + inline + bool + enabled() + { + return _enabled.load(std::memory_order_relaxed); + } + void enable(const bool); - bool enabled(); int get(const int who); int set(const int who, const int ioprio); @@ -33,6 +47,15 @@ namespace ioprio { static thread_local int thread_prio; - SetFrom(const pid_t pid); + [[gnu::always_inline]] + inline + SetFrom(const pid_t pid_) + { + if(ioprio::enabled()) + _slow_apply(pid_); + } + + private: + void _slow_apply(const pid_t); }; }; From 7ec6f8e2f2cd2b5cc56b203fb389fc1b9cc27e83 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Thu, 14 May 2026 07:53:15 -0500 Subject: [PATCH 15/23] Manage cloned /dev/fuse fds in sessions - Store cloned read-worker fds on fuse_session and close them there. - Set up clone fallback through session helpers used by the loop. - Carry per-request fds through receive, process, and reply paths. Tests show significant performance increases in highly concurrent workloads on a high core count system. Upto 15% better on reads and 27% on writes. --- vendored/libfuse/include/fuse_req.hpp | 1 + vendored/libfuse/lib/fuse_i.hpp | 15 +++- vendored/libfuse/lib/fuse_loop.cpp | 48 +++++++++---- vendored/libfuse/lib/fuse_lowlevel.cpp | 33 +++++---- vendored/libfuse/lib/fuse_session.cpp | 98 +++++++++++++++++++++++++- 5 files changed, 162 insertions(+), 33 deletions(-) diff --git a/vendored/libfuse/include/fuse_req.hpp b/vendored/libfuse/include/fuse_req.hpp index 7ba7df4c..e3d04d0c 100644 --- a/vendored/libfuse/include/fuse_req.hpp +++ b/vendored/libfuse/include/fuse_req.hpp @@ -10,6 +10,7 @@ struct fuse_req_t { fuse_req_ctx_t ctx; struct fuse_session *se; + int fd; fuse_conn_info_t conn; unsigned int ioctl_64bit : 1; }; diff --git a/vendored/libfuse/lib/fuse_i.hpp b/vendored/libfuse/lib/fuse_i.hpp index 8989aa07..fb395416 100644 --- a/vendored/libfuse/lib/fuse_i.hpp +++ b/vendored/libfuse/lib/fuse_i.hpp @@ -15,6 +15,7 @@ #include "extern_c.h" #include +#include /* Simplified fuse_session - fields inlined from former fuse_chan * Since mergerfs only supports one mount, we collapse the hierarchy: @@ -24,9 +25,11 @@ struct fuse_session { int (*receive_buf)(struct fuse_session *se, + int fd, fuse_msgbuf_t *msgbuf); void (*process_buf)(struct fuse_session *se, + int fd, const fuse_msgbuf_t *msgbuf); void (*destroy)(void *data); @@ -35,8 +38,10 @@ struct fuse_session std::atomic exited; /* Formerly in fuse_chan - inlined for single-mount simplicity */ - int fd; /* /dev/fuse file descriptor */ - size_t bufsize; /* Buffer size for I/O operations */ + int fd; /* /dev/fuse file descriptor */ + size_t bufsize; /* Buffer size for I/O operations */ + + std::vector clone_fds; /* cloned /dev/fuse fds for read workers */ }; struct fuse_notify_req @@ -50,6 +55,12 @@ struct fuse_notify_req struct fuse_notify_req *prev; }; +bool fuse_session_setup_read_fds(struct fuse_session *se, + int read_thread_count); + +int fuse_session_read_fd(struct fuse_session *se, + int index); + EXTERN_C_BEGIN diff --git a/vendored/libfuse/lib/fuse_loop.cpp b/vendored/libfuse/lib/fuse_loop.cpp index 408853b9..e6b3eec4 100644 --- a/vendored/libfuse/lib/fuse_loop.cpp +++ b/vendored/libfuse/lib/fuse_loop.cpp @@ -11,7 +11,6 @@ #include "maintenance_thread.hpp" #include "fuse_i.hpp" -#include "fuse_kernel.h" #include "fuse_lowlevel.h" #include "fuse_cfg.hpp" @@ -67,13 +66,16 @@ _nproc() struct AsyncWorker { fuse_session *_se; + int _fd; sem_t *_finished; std::shared_ptr _process_tp; AsyncWorker(fuse_session *se_, + int fd_, sem_t *finished_, std::shared_ptr process_tp_) : _se(se_), + _fd(fd_), _finished(finished_), _process_tp(process_tp_) { @@ -102,7 +104,7 @@ struct AsyncWorker while(true) { pthread_setcancelstate(PTHREAD_CANCEL_ENABLE,NULL); - rv = _se->receive_buf(_se,msgbuf); + rv = _se->receive_buf(_se,_fd,msgbuf); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,NULL); if(rv > 0) @@ -118,9 +120,9 @@ struct AsyncWorker } _process_tp->enqueue_work(ptok, - [=]() + [se = _se,fd = _fd,msgbuf]() { - _se->process_buf(_se,msgbuf); + se->process_buf(se,fd,msgbuf); msgbuf_free(msgbuf); }); } @@ -130,12 +132,15 @@ struct AsyncWorker struct SyncWorker { fuse_session *_se; + int _fd; sem_t *_finished; SyncWorker(fuse_session *se_, + int fd_, sem_t *finished_) : _se(se_), + _fd(fd_), _finished(finished_) { } @@ -162,7 +167,7 @@ struct SyncWorker while(true) { pthread_setcancelstate(PTHREAD_CANCEL_ENABLE,NULL); - rv = _se->receive_buf(_se,msgbuf); + rv = _se->receive_buf(_se,_fd,msgbuf); pthread_setcancelstate(PTHREAD_CANCEL_DISABLE,NULL); if(rv > 0) break; @@ -176,7 +181,7 @@ struct SyncWorker return ::_print_error(rv); } - _se->process_buf(_se,msgbuf); + _se->process_buf(_se,_fd,msgbuf); msgbuf_free(msgbuf); } @@ -255,15 +260,18 @@ fuse_session_loop_mt(struct fuse_session *se_, const std::string pin_threads_type_) { sem_t finished; + bool cloned_read_fds; int read_thread_count; int process_thread_count; int process_thread_queue_depth; std::vector read_threads; std::vector process_threads; - std::unique_ptr read_tp; - std::shared_ptr process_tp; sem_init(&finished,0,0); + DEFER{ sem_destroy(&finished); }; + + std::unique_ptr read_tp; + std::shared_ptr process_tp; read_thread_count = raw_read_thread_count_; process_thread_count = raw_process_thread_count_; @@ -274,21 +282,28 @@ fuse_session_loop_mt(struct fuse_session *se_, if(process_thread_count > 0) process_tp = std::make_shared(process_thread_count, - process_thread_queue_depth, - "fuse.process"); + process_thread_queue_depth, + "fuse.process"); + + cloned_read_fds = fuse_session_setup_read_fds(se_,read_thread_count); read_tp = std::make_unique(read_thread_count, - read_thread_count, - "fuse.read"); + read_thread_count, + "fuse.read"); if(process_tp) { for(auto i = 0; i < read_thread_count; i++) - read_tp->enqueue_work(AsyncWorker(se_,&finished,process_tp)); + read_tp->enqueue_work(AsyncWorker(se_, + fuse_session_read_fd(se_,i), + &finished, + process_tp)); } else { for(auto i = 0; i < read_thread_count; i++) - read_tp->enqueue_work(SyncWorker(se_,&finished)); + read_tp->enqueue_work(SyncWorker(se_, + fuse_session_read_fd(se_,i), + &finished)); } if(read_tp) @@ -301,10 +316,12 @@ fuse_session_loop_mt(struct fuse_session *se_, SysLog::info("read-thread-count={}; " "process-thread-count={}; " "process-thread-queue-depth={}; " + "fuse-dev-ioc-clone={}; " "pin-threads={};", read_thread_count, process_thread_count, process_thread_queue_depth, + cloned_read_fds, pin_threads_type_); while(!fuse_session_exited(se_)) @@ -313,7 +330,8 @@ fuse_session_loop_mt(struct fuse_session *se_, for(auto t : read_threads) pthread_cancel(t); - sem_destroy(&finished); + read_tp.reset(); + process_tp.reset(); return 0; } diff --git a/vendored/libfuse/lib/fuse_lowlevel.cpp b/vendored/libfuse/lib/fuse_lowlevel.cpp index cf79fd9a..d754fe85 100644 --- a/vendored/libfuse/lib/fuse_lowlevel.cpp +++ b/vendored/libfuse/lib/fuse_lowlevel.cpp @@ -103,16 +103,16 @@ iov_length(const struct iovec *iov, static int -fuse_send_msg(struct fuse_session *se, - struct iovec *iov, - int count) +fuse_send_msg(const int fd_, + struct iovec *iov, + int count) { int rv; struct fuse_out_header *out = (fuse_out_header*)iov[0].iov_base; out->len = iov_length(iov, count); - rv = writev(se->fd,iov,count); + rv = writev(fd_,iov,count); if(rv == -1) return -errno; @@ -144,7 +144,7 @@ fuse_send_reply_iov_nofree(fuse_req_t *req, iov[0].iov_base = &out; iov[0].iov_len = sizeof(struct fuse_out_header); - return fuse_send_msg(req->se, iov, count); + return fuse_send_msg(req->fd, iov, count); } static @@ -440,7 +440,7 @@ fuse_reply_data(fuse_req_t *req, if(fuse_cfg.debug) fuse_debug_data_out(req->ctx.unique,bufsize_); - res = fuse_send_msg(req->se,iov,2); + res = fuse_send_msg(req->fd,iov,2); fuse_req_free(req); return res; @@ -1310,7 +1310,7 @@ send_notify_iov(struct fuse_session *se, iov[0].iov_base = &out; iov[0].iov_len = sizeof(struct fuse_out_header); - return fuse_send_msg(se, iov, count); + return fuse_send_msg(se->fd, iov, count); } int @@ -1558,7 +1558,7 @@ fuse_ll_destroy(void *data) static void -fuse_send_errno(struct fuse_session *se_, +fuse_send_errno(const int fd_, const int errno_, const uint64_t unique_id_) { @@ -1574,25 +1574,26 @@ fuse_send_errno(struct fuse_session *se_, if(fuse_cfg.debug) fuse_debug_out_header(&out); - fuse_send_msg(se_,&iov,1); + fuse_send_msg(fd_,&iov,1); } static void -fuse_send_enomem(struct fuse_session *se_, +fuse_send_enomem(const int fd_, const uint64_t unique_id_) { - fuse_send_errno(se_,ENOMEM,unique_id_); + fuse_send_errno(fd_,ENOMEM,unique_id_); } static int fuse_ll_buf_receive_read(struct fuse_session *se_, + int fd_, fuse_msgbuf_t *msgbuf_) { int rv; - rv = read(se_->fd,msgbuf_->mem,msgbuf_->size); + rv = read(fd_,msgbuf_->mem,msgbuf_->size); if(rv == -1) return -errno; @@ -1608,6 +1609,7 @@ fuse_ll_buf_receive_read(struct fuse_session *se_, static void fuse_ll_buf_process_read(struct fuse_session *se_, + int fd_, const fuse_msgbuf_t *msgbuf_) { int err; @@ -1621,7 +1623,7 @@ fuse_ll_buf_process_read(struct fuse_session *se_, req = fuse_req_alloc(); if(req == NULL) - return fuse_send_enomem(se_,in->unique); + return fuse_send_enomem(fd_,in->unique); req->ctx.len = in->len; req->ctx.opcode = in->opcode; @@ -1633,6 +1635,7 @@ fuse_ll_buf_process_read(struct fuse_session *se_, req->ctx.umask = 0; req->conn = f.conn; req->se = se_; + req->fd = fd_; req->ioctl_64bit = 0; err = ENOSYS; @@ -1653,6 +1656,7 @@ fuse_ll_buf_process_read(struct fuse_session *se_, static void fuse_ll_buf_process_read_init(struct fuse_session *se_, + int fd_, const fuse_msgbuf_t *msgbuf_) { int err; @@ -1666,7 +1670,7 @@ fuse_ll_buf_process_read_init(struct fuse_session *se_, req = fuse_req_alloc(); if(req == NULL) - return fuse_send_enomem(se_,in->unique); + return fuse_send_enomem(fd_,in->unique); req->ctx.len = in->len; req->ctx.opcode = in->opcode; @@ -1678,6 +1682,7 @@ fuse_ll_buf_process_read_init(struct fuse_session *se_, req->ctx.pid = in->pid; req->ctx.umask = 0; req->se = se_; + req->fd = fd_; err = EIO; if(in->opcode != FUSE_INIT) diff --git a/vendored/libfuse/lib/fuse_session.cpp b/vendored/libfuse/lib/fuse_session.cpp index 4e8f682c..e63db49d 100644 --- a/vendored/libfuse/lib/fuse_session.cpp +++ b/vendored/libfuse/lib/fuse_session.cpp @@ -8,16 +8,24 @@ #include "fuse_i.hpp" #include "fuse_kernel.h" +#include "syslog.hpp" #include #include +#include #include #include #include +#include #include +#include #include #include +#ifndef O_CLOEXEC +#define O_CLOEXEC 0 +#endif + /* * Simplified session management for single-mount filesystems. * The fuse_chan structure has been removed and its fields inlined @@ -25,6 +33,42 @@ * one mount point per process. */ +static +int +_clone_fuse_fd(const int fd_) +{ + int rv; + int clone_fd; + uint32_t source_fd; + + clone_fd = ::open("/dev/fuse",O_RDWR|O_CLOEXEC); + if(clone_fd == -1) + return -errno; + + source_fd = fd_; + rv = ::ioctl(clone_fd,FUSE_DEV_IOC_CLONE,&source_fd); + if(rv == -1) + { + int err = errno; + ::close(clone_fd); + return -err; + } + + return clone_fd; +} + +static +void +_close_clone_fds(struct fuse_session *se_) +{ + for(auto fd : se_->clone_fds) + { + if(fd != -1) + ::close(fd); + } + se_->clone_fds.clear(); +} + struct fuse_session* fuse_session_new(void *data, void *receive_buf, @@ -40,8 +84,8 @@ fuse_session_new(void *data, } se->f = static_cast(data); - se->receive_buf = reinterpret_cast(receive_buf); - se->process_buf = reinterpret_cast(process_buf); + se->receive_buf = reinterpret_cast(receive_buf); + se->process_buf = reinterpret_cast(process_buf); se->destroy = reinterpret_cast(destroy); se->fd = -1; @@ -57,6 +101,8 @@ fuse_session_destroy(struct fuse_session *se) if(se->destroy) se->destroy(se->f); + ::_close_clone_fds(se); + if(se->fd != -1) { close(se->fd); @@ -93,6 +139,8 @@ fuse_session_data(struct fuse_session *se) int fuse_session_clearfd(struct fuse_session *se) { + ::_close_clone_fds(se); + int fd = se->fd; se->fd = -1; return fd; @@ -102,9 +150,55 @@ void fuse_session_setfd(struct fuse_session *se, int fd) { + ::_close_clone_fds(se); + se->fd = fd; } +bool +fuse_session_setup_read_fds(struct fuse_session *se_, + const int read_thread_count_) +{ + ::_close_clone_fds(se_); + + if(read_thread_count_ <= 1) + return false; + + se_->clone_fds.reserve(read_thread_count_ - 1); + + for(auto i = 1; i < read_thread_count_; i++) + { + int fd; + + fd = ::_clone_fuse_fd(se_->fd); + if(fd < 0) + { + int err = -fd; + + ::_close_clone_fds(se_); + SysLog::warning("failed to clone /dev/fuse fd for read threads; " + "using shared fd: {} ({})", + strerror(err), + err); + return false; + } + + se_->clone_fds.push_back(fd); + } + + return true; +} + +int +fuse_session_read_fd(struct fuse_session *se_, + const int index_) +{ + if((index_ > 0) && (index_ <= (int)se_->clone_fds.size())) + return se_->clone_fds[index_ - 1]; + + return se_->fd; +} + void fuse_session_setbufsize(struct fuse_session *se, size_t bufsize) From 89b4046d11ae530e6c1bb127ea34ff4ba6099a58 Mon Sep 17 00:00:00 2001 From: shyney7 Date: Sun, 17 May 2026 01:31:38 +0200 Subject: [PATCH 16/23] docs/faq: add SELinux relabeling entry --- .../docs/faq/compatibility_and_integration.md | 64 +++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/mkdocs/docs/faq/compatibility_and_integration.md b/mkdocs/docs/faq/compatibility_and_integration.md index c0db9a5a..1b82d9b4 100644 --- a/mkdocs/docs/faq/compatibility_and_integration.md +++ b/mkdocs/docs/faq/compatibility_and_integration.md @@ -149,6 +149,70 @@ It is generally recommended to not use user namespacing / id mapping given the complication it introduces. +## Does mergerfs work with SELinux relabeling? + +FUSE filesystems, including mergerfs, do not support per-inode +SELinux labels (see [Red Hat bug +1272868](https://bugzilla.redhat.com/show_bug.cgi?id=1272868) and the +[upstream fuse-devel +thread](https://sourceforge.net/p/fuse/mailman/fuse-devel/thread/CAN9vWDLGA4%3Dg3peKVWkX7gLRiVUUvJAQx9kiXHgqdeX6eh0MYA%40mail.gmail.com/)). +When a container runtime such as Podman or Docker is asked to relabel +a bind mount with the `:z` or `:Z` mount option, it first probes the +source for `security.selinux` xattr support. On a FUSE path that +probe returns `EOPNOTSUPP` and the runtime skips relabeling entirely. +Files inside the mount keep the `fusefs_t` type assigned by the +kernel. No tree walk is performed and no journal noise is produced at +default verbosity. The runtime logs `Labeling not supported on +""` only at debug level (for example +`podman --log-level=debug`). The flag is therefore a no-op against a +mergerfs mount; not harmful, but misleading. + +Containers can still read and write a mergerfs pool without `:z` +because two independent access checks must succeed and both can be +satisfied without any relabel: + +* **SELinux:** on Fedora, RHEL / CentOS Stream, and SUSE the shipped + [container-selinux](https://github.com/containers/container-selinux) + policy already permits the `container_t` process type to manage + files of type `fusefs_t`. See `container_selinux(8)`. +* **DAC:** the in-container UID and GID must still be able to read + and write the files on the underlying branches. With rootless + Podman this is most easily arranged by aligning the container UID + with the host owner of the branches (for example + `--userns=keep-id`, or Quadlet `UserNS=keep-id`). + +The recommendation is to omit `:z` and `:Z` on bind mounts whose +source is a mergerfs (or any other FUSE) path, and to keep them on +bind mounts whose source is a regular local filesystem (ext4, btrfs, +xfs, etc.) where relabeling does real work. On distributions whose +container-selinux policy does not already include the `fusefs_t` +allow rule, the targeted fix is to set a single label at mount time +by adding `context="system_u:object_r:container_file_t:s0"` to the +mergerfs mount options (for example in `/etc/fstab`). Files inside +the pool then appear as `container_file_t`, which container-selinux +permits `container_t` to manage everywhere. See [Red Hat's mounting +file systems +guide](https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/selinux_users_and_administrators_guide/sect-security-enhanced_linux-working_with_selinux-mounting_file_systems). +This is preferred over the much broader +`--security-opt label=disable`, which removes SELinux confinement +from the container as a whole. The trade-off of `context=` is that +host-side processes touching the mount (rsync, backup tools, web +UIs, cron jobs) see `container_file_t` rather than `fusefs_t` and +may require a small SELinux policy adjustment. + +To verify on your own system: + +``` +podman --log-level=debug run --rm \ + -v /path/to/mergerfs:/data:z \ + alpine true 2>&1 | grep -iE 'relabel|setfilecon|Labeling' +``` + +Expected output is a single line of the form +`Labeling not supported on "/path/to/mergerfs"` and nothing in +`ausearch -m AVC,SELINUX_ERR` for the mount. + + ## Can mergerfs be used with NFS root squash? If mergerfs is pooling a NFS mount then root squash must be disabled From 42ab831d97e31e9edc674d339db247e8239d7912 Mon Sep 17 00:00:00 2001 From: shyney7 Date: Sun, 17 May 2026 01:31:39 +0200 Subject: [PATCH 17/23] docs/setup: link install :z example to SELinux FAQ --- mkdocs/docs/setup/installation.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mkdocs/docs/setup/installation.md b/mkdocs/docs/setup/installation.md index 8de89d0c..0f6a41ab 100644 --- a/mkdocs/docs/setup/installation.md +++ b/mkdocs/docs/setup/installation.md @@ -139,7 +139,10 @@ docker run --device=/dev/fuse --cap-add=SYS_ADMIN -v /mnt/to-merge:/mnt/to-merge * `--cap-add=SYS_ADMIN`: Give proper permissions to mount. * `-v /mnt/mergerfs:/mnt/mergerfs:z,shared`: `z` and `shared` will allow the mount to be shared between the container and host and - therefore other containers. + therefore other containers. Note that once mergerfs is mounted + inside the container the resulting FUSE filesystem does not + support SELinux relabeling. See [Does mergerfs work with SELinux + relabeling?](../faq/compatibility_and_integration.md#does-mergerfs-work-with-selinux-relabeling) * `-v /mnt/to-merge:/mnt/to-merge:rshared`: The mounts from the host to merge within the container. From 8ea47df13e50c8093c4854d7e726f59b2c565009 Mon Sep 17 00:00:00 2001 From: natalefalco <31706045+natalefalco@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:09:51 +0200 Subject: [PATCH 18/23] Include Docker Compose setup for mergerfs Added Docker Compose configuration for running mergerfs. --- mkdocs/docs/setup/installation.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/mkdocs/docs/setup/installation.md b/mkdocs/docs/setup/installation.md index 0f6a41ab..6b59a6e3 100644 --- a/mkdocs/docs/setup/installation.md +++ b/mkdocs/docs/setup/installation.md @@ -135,6 +135,25 @@ docker pull ghcr.io/trapexit/mergerfs:TAG docker run --device=/dev/fuse --cap-add=SYS_ADMIN -v /mnt/to-merge:/mnt/to-merge:rshared -v /mnt/mergerfs:/mnt/mergerfs:z,shared ghcr.io/trapexit/mergerfs:TAG -f '/mnt/to-merge/*' '/mnt/mergerfs' ``` + or with docker compose + +``` +services: + mergerfs: + image: ghcr.io/trapexit/mergerfs:TAG + container_name: mergerfs + restart: unless-stopped + privileged: true + cap_add: + - SYS_ADMIN + devices: + - /dev/fuse:/dev/fuse + command: -f '/mnt/to-merge/*' '/mnt/mergerfs' + volumes: + - /mnt/to-merge:/mnt/to-merge:rshared + - /mnt/mergerfs:/mnt/mergerfs:z,shared +``` + * `--device=/dev/fuse`: Pass in host FUSE device. * `--cap-add=SYS_ADMIN`: Give proper permissions to mount. * `-v /mnt/mergerfs:/mnt/mergerfs:z,shared`: `z` and `shared` will From 515b0549ef615418df41ed3ffd993d036bc52e1f Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Sat, 5 Sep 2026 14:58:23 -0500 Subject: [PATCH 19/23] docs/performance: explain mergerfs IO performance vs RAID Describe why mergerfs does not aggregate branch performance like RAID/ZFS (whose aggregate throughput is also capped by shared interconnects) and instead provides independent per-branch performance: best case A+B+C across concurrent streams, worst case single-stream limited to one branch's speed. --- mkdocs/docs/performance.md | 59 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/mkdocs/docs/performance.md b/mkdocs/docs/performance.md index 802e302a..b0887513 100644 --- a/mkdocs/docs/performance.md +++ b/mkdocs/docs/performance.md @@ -48,6 +48,65 @@ before changing them to understand how functionality will change. * use LVM and LVM cache to place a SSD in front of your HDDs +## Understanding mergerfs IO Performance + +A common source of confusion when evaluating **mergerfs**' performance +is comparing it to traditional RAID or filesystem level pooling +technologies such as mdadm, ZFS, btrfs, or striped LVM. Those +solutions aggregate devices into a single virtual device and spread IO +across all members. Under favorable conditions that means the +performance of the pool is the sum of the performance of its members: +striped reads and writes hit multiple drives at once and the aggregate +throughput scales with the number of devices, up to the limits of the +interconnects (PCIe, SATA, network, etc.) which every member must +share. + +**mergerfs** does not aggregate or stripe data. Every file lives +wholly on one branch and **mergerfs** simply routes access to it. As a +result **mergerfs** does not aggregate the performance of its branches +in the way RAID does. What it provides instead is **independent +performance across branches**. + +* **Best case:** workloads which access different branches + concurrently perform independently of one another. If you have 3 + filesystems on 3 drives and each is being actively used, the total + throughput can be A + B + C: the sum of each branch performing at + full speed. No single drive is a bottleneck for the others. +* **Worst case:** a single stream of IO - one process reading or + writing one file - performs at the speed of the single branch the + file is on, no differently than a lone filesystem. Additional + branches do not make a single file faster. + +This means that when reasoning about expected performance the key +question is whether your workload is **single stream** or **multi +stream**: + +* Single large file transfers, video playback, or one process doing + sequential IO will never exceed the speed of one branch regardless + of how many are mounted. If a single stream must be faster, put a + faster device (tiered cache, SSD, LVM cache) in front of the + branches or use an actual striped RAID layer beneath **mergerfs**' + branches. +* Concurrency across branches is where **mergerfs** shines. Many + parallel downloads, seeding torrents, multiple rsync streams, + multiple applications reading different files: these naturally + spread across branches and their throughput adds up. + +There are practical upsides to this non-aggregated design as well: + +* Drives of different sizes and speeds don't reduce the whole pool to + the slowest member. Each branch performs at its own speed. +* The failure of one drive only affects the files on that branch + rather than the integrity of the entire pool. + +Finally, keep in mind that **mergerfs** is a proxy above the branches, +so measured throughput is the lesser of the branch's native performance +and any overhead introduced by **mergerfs** and FUSE. Use +[nullrw](config/options.md) or a ram disk as a branch to measure that +overhead separately, and see the [benchmarking +section](benchmarking.md) for how to benchmark meaningfully. + + ## Additional Reading * [Benchmarking](benchmarking.md) From cfb925e5ae5da37be7e3356059516b63482c4fa7 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Mon, 21 Sep 2026 10:01:04 -0500 Subject: [PATCH 20/23] docs: clarify branch resolution and mount ordering --- mkdocs/docs/config/branches-mount-timeout.md | 26 +++++++ mkdocs/docs/config/branches.md | 37 ++++++++-- mkdocs/docs/config/rename-exdev.md | 3 +- mkdocs/docs/faq/why_isnt_it_working.md | 2 +- mkdocs/docs/index.md | 2 +- mkdocs/docs/known_issues_bugs.md | 6 +- mkdocs/docs/project_comparisons.md | 4 +- mkdocs/docs/quickstart.md | 76 ++++++++++++++++---- mkdocs/docs/runtime_interface.md | 10 +-- mkdocs/docs/tooling.md | 7 +- 10 files changed, 138 insertions(+), 35 deletions(-) diff --git a/mkdocs/docs/config/branches-mount-timeout.md b/mkdocs/docs/config/branches-mount-timeout.md index c2f35ea8..3f28b5d8 100644 --- a/mkdocs/docs/config/branches-mount-timeout.md +++ b/mkdocs/docs/config/branches-mount-timeout.md @@ -38,6 +38,32 @@ These files and xattrs are used to help `mergerfs` know whether or not a directory happens to be a branch or is the underlying directory mount point. +## Paths which appear after startup + +The [`branches` resolution process](branches.md#resolution) runs before +this option starts waiting. A missing literal path is retained as a +branch, so a configuration such as `/mnt/hdd/disk0/foobar` can work +when `/mnt/hdd/disk0` is mounted asynchronously and `foobar` only +becomes visible afterward, provided the path then meets one of the +"mounted" conditions above. + +For each branch which is not ready, mergerfs first runs `mount` once +with the full branch path as the target and then polls that same path +until the timeout expires. It does not create the path, try to mount +its parent, or rerun `mount`. In the example above, the initial +`mount /mnt/hdd/disk0/foobar` may fail while the path is absent. If +another service mounts `/mnt/hdd/disk0` and +`/mnt/hdd/disk0/foobar` appears and meets one of the "mounted" +conditions before the deadline, mergerfs detects it during polling +and continues. + +Globs are different. If `/mnt/hdd/disk0/*` has no matches during +branch resolution, the literal path `/mnt/hdd/disk0/*` is retained. +The mount wait does not expand it again after `/mnt/hdd/disk0` +becomes available. Ensure the parent filesystem is mounted before +mergerfs, configure a literal branch path, or update `branches` +through the runtime API once the paths exist. + **NOTE:** If on a `systemd` based system and using `fstab` it is a good idea to set the mount option diff --git a/mkdocs/docs/config/branches.md b/mkdocs/docs/config/branches.md index 2f2a5293..f51f1cdf 100644 --- a/mkdocs/docs/config/branches.md +++ b/mkdocs/docs/config/branches.md @@ -14,7 +14,7 @@ Branches currently have two options which can be set. A included in a policy calculation and a individual [minfreespace](#minfreespace) value. The values are set by prepending an `=` at the end of a branch designation and using commas -as delimiters. Example: `/mnt/drive=RW,1234` +as delimiters. Example: `/mnt/hdd/disk0=RW,1234` ### branch mode @@ -34,6 +34,33 @@ Same purpose and syntax as the [global option](minfreespace.md) but specific to the branch. Defaults to the global value. +### resolution + +Each colon-delimited branch designation is resolved independently when +the `branches` option is parsed, both at startup and when it is updated +through the runtime API: + +1. The path portion is expanded as a glob. +2. If the glob matches one or more directories, each match becomes a + branch. +3. If the glob has no matches, mergerfs logs a notice and keeps the + configured path unchanged as a single, literal branch. +4. mergerfs resolves each resulting path with `realpath`. If that + succeeds, the canonical path is stored. If it fails, typically + because the path does not exist yet, the unchanged path is stored. +5. A path which exists but is not a directory is logged and skipped. A + path which does not exist is logged but retained. + +Consequently, a literal branch such as `/mnt/hdd/disk0/foobar` may be +configured before it exists and can become usable after +`/mnt/hdd/disk0` is mounted. Branch resolution is not repeated in the +background, however. If `/mnt/hdd/disk0/*` has no matches when it is +resolved, mergerfs stores the literal path `/mnt/hdd/disk0/*`; +directories which match it later are not discovered automatically. +Update `branches` through the runtime API or restart mergerfs to +resolve the glob again. + + ### globbing To make it easier to include multiple branches mergerfs supports @@ -42,18 +69,18 @@ MUST be escaped when using via the shell else the shell itself will apply the glob itself.** ``` -# mergerfs /mnt/hdd\*:/mnt/ssd /media +# mergerfs /mnt/hdd/\*:/mnt/ssd/\* /media ``` -The above line will use all directories in /mnt prefixed with **hdd** -as well as **ssd**. +The above line will use all directories within `/mnt/hdd` and +`/mnt/ssd`. To have the pool mounted at boot or otherwise accessible from related tools use `/etc/fstab`. ``` # -/mnt/hdd*:/mnt/ssd /media mergerfs minfreespace=16G 0 0 +/mnt/hdd/*:/mnt/ssd/* /media mergerfs minfreespace=16G 0 0 ``` **NOTE:** The globbing is done at mount or when updated using the diff --git a/mkdocs/docs/config/rename-exdev.md b/mkdocs/docs/config/rename-exdev.md index 203e053d..20773d42 100644 --- a/mkdocs/docs/config/rename-exdev.md +++ b/mkdocs/docs/config/rename-exdev.md @@ -2,7 +2,8 @@ If using path preservation and a `rename` fails with `EXDEV`: -1. Move file from `/branch/a/b/c` to `/branch/.mergerfs_rename_exdev/a/b/c`. +1. Move file from `/mnt/hdd/disk0/a/b/c` to + `/mnt/hdd/disk0/.mergerfs_rename_exdev/a/b/c`. 2. symlink the rename's `newpath` to the moved file. The `target` value is determined by the value of `rename-exdev`. diff --git a/mkdocs/docs/faq/why_isnt_it_working.md b/mkdocs/docs/faq/why_isnt_it_working.md index 0bf5f819..bc4f46b8 100644 --- a/mkdocs/docs/faq/why_isnt_it_working.md +++ b/mkdocs/docs/faq/why_isnt_it_working.md @@ -81,7 +81,7 @@ where it was created. $ sudo touch /mnt/mergerfs/new-file $ sudo getfattr -n user.mergerfs.allpaths /mnt/mergerfs/new-file # file: mnt/mergerfs/new-file -user.mergerfs.allpaths="/mnt/hdd/drive1/new-file" +user.mergerfs.allpaths="/mnt/hdd/disk0/new-file" ``` If the location of the file is where it should be according to the diff --git a/mkdocs/docs/index.md b/mkdocs/docs/index.md index 4a293998..ce9558e7 100644 --- a/mkdocs/docs/index.md +++ b/mkdocs/docs/index.md @@ -105,7 +105,7 @@ The way in which mergerfs behaves is controlled by the ``` A + B = C -/disk1 /disk2 /merged +/mnt/hdd/disk0 /mnt/hdd/disk1 /media | | | +-- /dir1 +-- /dir1 +-- /dir1 | | | | | | diff --git a/mkdocs/docs/known_issues_bugs.md b/mkdocs/docs/known_issues_bugs.md index 8817c5e8..58833d4f 100644 --- a/mkdocs/docs/known_issues_bugs.md +++ b/mkdocs/docs/known_issues_bugs.md @@ -92,15 +92,15 @@ If you always want the directory information from the one with the most recent mtime then use the `newest` policy for `getattr`. -### 'mv /mnt/pool/foo /mnt/disk1/foo' removes 'foo' +### 'mv /mnt/pool/foo /mnt/hdd/disk0/foo' removes 'foo' This is not a bug. Run in verbose mode to better understand what's happening: ``` -$ mv -v /mnt/pool/foo /mnt/disk1/foo -copied '/mnt/pool/foo' -> '/mnt/disk1/foo' +$ mv -v /mnt/pool/foo /mnt/hdd/disk0/foo +copied '/mnt/pool/foo' -> '/mnt/hdd/disk0/foo' removed '/mnt/pool/foo' $ ls /mnt/pool/foo ls: cannot access '/mnt/pool/foo': No such file or directory diff --git a/mkdocs/docs/project_comparisons.md b/mkdocs/docs/project_comparisons.md index 8fb2eb01..4e6ef4f5 100644 --- a/mkdocs/docs/project_comparisons.md +++ b/mkdocs/docs/project_comparisons.md @@ -23,9 +23,9 @@ and offers better performance. As of 2020, the author of mhddfs has Below is an example of mhddfs and mergerfs setup to work similarly. ``` -mhddfs -o mlimit=4G,allow_other /mnt/drive1,/mnt/drive2 /mnt/pool +mhddfs -o mlimit=4G,allow_other /mnt/hdd/disk0,/mnt/hdd/disk1 /mnt/pool -mergerfs -o minfreespace=4G,category.create=ff /mnt/drive1:/mnt/drive2 /mnt/pool +mergerfs -o minfreespace=4G,category.create=ff /mnt/hdd/disk0:/mnt/hdd/disk1 /mnt/pool ``` diff --git a/mkdocs/docs/quickstart.md b/mkdocs/docs/quickstart.md index 8db95bc6..374d97c8 100644 --- a/mkdocs/docs/quickstart.md +++ b/mkdocs/docs/quickstart.md @@ -66,24 +66,69 @@ branches](config/branches.md#branch-setup). ``` # Two positional options: `:` colon separated branches and mountpoint -mergerfs -o opt[,opt...] /branch0:/branch1 /mountpoint +mergerfs -o opt[,opt...] /mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media # or list of positional options where the last is the mountpoint -mergerfs -o opt[,opt...] /branch0 /branch1 /mountpoint +mergerfs -o opt[,opt...] /mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2 /media # or set mountpoint via -o mountpoint -mergerfs -o mountpoint=/mountpoint /branch0 /branch1 +mergerfs -o mountpoint=/media /mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2 ``` ``` -mergerfs -o cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false /mnt/hdd0:/mnt/hdd1 /media +mergerfs -o cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false /mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media ``` ### /etc/fstab ``` -/mnt/hdd0:/mnt/hdd1 /media mergerfs cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false 0 0 +/mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media mergerfs cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false 0 0 ``` +### systemd mount dependencies + +On a `systemd` based system, no custom service or setup script is +needed when the branch filesystems and mergerfs are all configured in +`/etc/fstab`. Add +[`x-systemd.requires-mounts-for=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.mount.html#x-systemd.wants-mounts-for=) +once for each branch mount target in the mergerfs entry: + +```fstab +LABEL=disk0 /mnt/hdd/disk0 ext4 defaults 0 2 +LABEL=disk1 /mnt/hdd/disk1 ext4 defaults 0 2 +LABEL=disk2 /mnt/hdd/disk2 ext4 defaults 0 2 +/mnt/hdd/disk0:/mnt/hdd/disk1:/mnt/hdd/disk2 /media mergerfs cache.files=off,category.create=pfrd,func.getattr=newest,dropcacheonclose=false,x-systemd.requires-mounts-for=/mnt/hdd/disk0,x-systemd.requires-mounts-for=/mnt/hdd/disk1,x-systemd.requires-mounts-for=/mnt/hdd/disk2 0 0 +``` + +Replace the example labels and filesystem types with those used by the +system. `systemd` creates `Requires=` and `After=` dependencies from +the mergerfs mount to `mnt-hdd-disk0.mount`, +`mnt-hdd-disk1.mount`, and `mnt-hdd-disk2.mount`. It starts those +mounts first and does not start the mergerfs mount if a required mount +fails. This checks the mount units, not merely whether the +`/mnt/hdd/diskN` directories exist. + +After editing `/etc/fstab`, reload the generated units: + +```shell +sudo systemctl daemon-reload +``` + +The dependencies apply the next time `/media` is mounted and on +subsequent boots. + +When using a service such as the +[`mergerfs-media.service`](#systemd-simple) example below instead of an +`/etc/fstab` mergerfs entry, add the equivalent directive to its +`[Unit]` section: + +```systemd +RequiresMountsFor=/mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2 +``` + +`RequiresMountsFor=` adds both the requirement and ordering +dependencies, so separate `Requires=` and `After=` entries for these +mounts are unnecessary. + ### /etc/fstab w/ config file For more complex setups it can be useful to separate out the config. @@ -114,10 +159,9 @@ resolve the symlinks and use the real path. `ls -lh /etc/mergerfs/branches/media/*` ```text -lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd00 -> /mnt/hdd/hdd00 -lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd01 -> /mnt/hdd/hdd01 -lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd02 -> /mnt/hdd/hdd02 -lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd03 -> /mnt/hdd/hdd03 +lrwxrwxrwx 1 root root 14 Aug 4 2023 disk0 -> /mnt/hdd/disk0 +lrwxrwxrwx 1 root root 14 Aug 4 2023 disk1 -> /mnt/hdd/disk1 +lrwxrwxrwx 1 root root 14 Aug 4 2023 disk2 -> /mnt/hdd/disk2 ``` ### systemd (simple) @@ -127,6 +171,7 @@ lrwxrwxrwx 1 root root 21 Aug 4 2023 hdd03 -> /mnt/hdd/hdd03 ```systemd title="mergerfs-media.service" linenums="1" [Unit] Description=mergerfs /media service +RequiresMountsFor=/mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2 After=local-fs.target network.target [Service] @@ -138,8 +183,9 @@ ExecStart=/usr/bin/mergerfs \ -o category.create=pfrd \ -o func.getattr=newest \ -o dropcacheonclose=false \ - /mnt/hdd0 \ - /mnt/hdd1 \ + /mnt/hdd/disk0 \ + /mnt/hdd/disk1 \ + /mnt/hdd/disk2 \ /media ExecStop=/usr/bin/umount /media # Or if you need fusermount @@ -193,8 +239,9 @@ WantedBy=default.target ```systemd title="mergerfs-media.service" linenums="1" [Unit] Description=mergerfs /media service +RequiresMountsFor=/mnt/hdd/disk0 /mnt/hdd/disk1 /mnt/hdd/disk2 Requires=setup-for-mergerfs.service -After=local-fs.target network.target prepare-for-mergerfs.service +After=local-fs.target network.target setup-for-mergerfs.service [Service] Type=simple @@ -205,8 +252,9 @@ ExecStart=/usr/bin/mergerfs \ -o category.create=pfrd \ -o func.getattr=newest \ -o dropcacheonclose=false \ - /mnt/hdd0 \ - /mnt/hdd1 \ + /mnt/hdd/disk0 \ + /mnt/hdd/disk1 \ + /mnt/hdd/disk2 \ /media ExecStop=/usr/bin/umount /media # Or if you need fusermount diff --git a/mkdocs/docs/runtime_interface.md b/mkdocs/docs/runtime_interface.md index 8ac61040..3230cbf4 100644 --- a/mkdocs/docs/runtime_interface.md +++ b/mkdocs/docs/runtime_interface.md @@ -110,7 +110,7 @@ situation and continue to be able to access the xattr interface. ``` [trapexit:/mnt/mergerfs] $ getfattr -d .mergerfs -user.mergerfs.branches="/mnt/a=RW:/mnt/b=RW" +user.mergerfs.branches="/mnt/hdd/disk0=RW:/mnt/hdd/disk1=RW" user.mergerfs.minfreespace="4294967295" user.mergerfs.moveonenospc="false" ... @@ -122,9 +122,9 @@ user.mergerfs.category.search="mfs" [trapexit:/mnt/mergerfs] $ getfattr -n user.mergerfs.category.create .mergerfs user.mergerfs.category.search="prfd" -[trapexit:/mnt/mergerfs] $ setfattr -n user.mergerfs.branches -v '+ Date: Tue, 22 Sep 2026 08:30:35 -0500 Subject: [PATCH 21/23] docs: rework and expand project comparisons --- mkdocs/docs/project_comparisons.md | 277 ++++++++++++++++------------- mkdocs/docs/related_projects.md | 12 +- 2 files changed, 164 insertions(+), 125 deletions(-) diff --git a/mkdocs/docs/project_comparisons.md b/mkdocs/docs/project_comparisons.md index 4e6ef4f5..4be8d4aa 100644 --- a/mkdocs/docs/project_comparisons.md +++ b/mkdocs/docs/project_comparisons.md @@ -2,6 +2,11 @@ There are no solutions, only trade-offs. +This page includes both direct pooling alternatives and adjacent storage +technologies that solve different problems. Compare their operating layer, +failure domain, redundancy model, and platform requirements rather than only +their apparent pool capacity. + ## mhddfs @@ -15,14 +20,17 @@ and security issues. mergerfs provides a superset of mhddfs' features and offers better performance. As of 2020, the author of mhddfs has [moved to using mergerfs.](https://romanrm.net/mhddfs#update) -* **Better than mergerfs for:** legacy setups that already depend on - mhddfs-specific behavior and do not need newer features. +* **Better than mergerfs for:** Nothing? * **Worse than mergerfs for:** most modern deployments due to maintenance status, feature coverage, and reliability. -Below is an example of mhddfs and mergerfs setup to work similarly. +Below is an example of mhddfs and mergerfs set up to behave similarly. +mhddfs falls back to the drive with the most free space when every drive +has less than `mlimit` free; mergerfs with +[minfreespace](config/minfreespace.md) filters those branches instead and +may fail the create if none qualify. -``` +```sh mhddfs -o mlimit=4G,allow_other /mnt/hdd/disk0,/mnt/hdd/disk1 /mnt/pool mergerfs -o minfreespace=4G,category.create=ff /mnt/hdd/disk0:/mnt/hdd/disk1 /mnt/pool @@ -46,10 +54,11 @@ available in most Linux distros, making it harder to get installed for the average user. Development has been largely dormant for years. While aufs can often offer better peak performance due to being -primarily kernel-based (at least when `passthrough.io` is disabled in -mergerfs), mergerfs provides more configurability and is generally -easier to use. mergerfs, however, does not offer the overlay / -copy-on-write (CoW) features which aufs has. +primarily kernel-based (at least when +[passthrough.io](config/passthrough.md) is disabled in mergerfs), +mergerfs provides more configurability and is generally easier to use. +mergerfs, however, does not offer the overlay / copy-on-write (CoW) +features which aufs has. * **Better than mergerfs for:** kernel-level overlay/CoW workloads where that behavior is required. @@ -92,9 +101,10 @@ overlay/CoW-style layering, while mergerfs' primary feature is policy-based pooling and placement across multiple regular writable filesystems. -Since unionfs-fuse, as the name suggests, is a FUSE-based technology -it can be used without elevated privileges that kernel solutions such -as unionfs, aufs, and overlayfs require. +As a FUSE filesystem, unionfs-fuse can often be mounted by an +unprivileged user when the system's FUSE configuration permits it. Kernel +solutions such as unionfs, aufs, and overlayfs normally require additional +privileges. * **Better than mergerfs for:** overlay-style unions with copy-on-write semantics in userspace. @@ -106,10 +116,10 @@ as unionfs, aufs, and overlayfs require. * [https://docs.kernel.org/filesystems/overlayfs.html](https://docs.kernel.org/filesystems/overlayfs.html) -overlayfs is effectively the functional successor to unionfs, -unionfs-fuse, and aufs and is widely used by Linux container platforms -such as Docker and Podman. Both overlayfs and FUSE were originally -created by Miklos Szeredi. +overlayfs is effectively the successor to unionfs, unionfs-fuse, and +aufs and is widely used by Linux container platforms such as Docker and +Podman. Both overlayfs and FUSE were originally created by Miklos +Szeredi. If your use case is layering a writable filesystem on top of read-only filesystems then you should look first to overlayfs. Its feature set @@ -121,6 +131,24 @@ however is very different from mergerfs and solves different problems. flexible branch selection policies across many existing filesystems. +## fuse-overlayfs + +* [https://github.com/containers/fuse-overlayfs](https://github.com/containers/fuse-overlayfs) + +fuse-overlayfs implements overlayfs-style lower, upper, and work directories +in userspace. Its primary use case is rootless containers, where a kernel +overlayfs mount may not be available. + +Like unionfs-fuse and kernel overlayfs, it provides layering and copy-up +semantics rather than mergerfs-style placement across a pool of writable +filesystems. + +* **Better than mergerfs for:** rootless container and other unprivileged + overlay workloads. +* **Worse than mergerfs for:** general storage pooling across independent + writable filesystems. + + ## RAID0, JBOD, SPAN, drive concatenation, striping * [RAID0](https://en.wikipedia.org/wiki/Standard_RAID_levels#RAID_0) @@ -128,6 +156,10 @@ however is very different from mergerfs and solves different problems. * [SPAN](https://en.wikipedia.org/wiki/Non-RAID_drive_architectures#Concatenation_(SPAN,_BIG)) * [striping](https://en.wikipedia.org/wiki/Data_striping) +"JBOD" is ambiguous: it can also mean disks exposed independently with +no aggregation. In this section it refers only to concatenated or spanned +layouts. + These are block device technologies which, in some form, aggregate devices into what appears to be a single device on which a traditional filesystem can be used. The filesystem has no understanding of the @@ -162,12 +194,12 @@ allow the pool to continue to run should one device fail and rebuild the data once the device is replaced. mergerfs offers no parity or redundancy features so in that regard the -technologies are not comparable. [SnapRAID](https://www.snapraid.it) -or [NonRAID](https://github.com/qvr/nonraid) can be used in -combination with mergerfs to provide redundancy. Unlike traditional -RAID5 or RAID6 SnapRAID works with drives of different sizes and can -have more than 2 parity drives. Parity calculations with SnapRAID are -not done in real-time but NonRAID's are. +technologies are not comparable. [SnapRAID](https://www.snapraid.it) or +NonRAID (see [below](#nonraid)) can be used in combination with mergerfs +to provide redundancy. Unlike traditional RAID5 or RAID6, SnapRAID works +with drives of different sizes and can have more than 2 parity drives. +Parity calculations with SnapRAID are not done in real-time but +NonRAID's are. For more details and comparison of SnapRAID to related technologies see [https://www.snapraid.it/compare](https://www.snapraid.it/compare). @@ -204,7 +236,7 @@ data on that disk. ## mdadm * [https://en.wikipedia.org/wiki/Mdadm](https://en.wikipedia.org/wiki/Mdadm) -* [https://raid.wiki.kernel.org/index.php/A_guide_to_mdadm](https://raid.wiki.kernel.org/index.php/A_guide_to_mdadm) +* [https://docs.kernel.org/admin-guide/md.html](https://docs.kernel.org/admin-guide/md.html) `mdadm` is Linux userspace tooling for configuring and managing software RAID arrays provided by the kernel `md` driver. It can build RAID0/1/4/5/6/10 @@ -226,7 +258,7 @@ be combined with tools like SnapRAID or NonRAID when redundancy is needed. ## device mapper * [https://en.wikipedia.org/wiki/Device_mapper](https://en.wikipedia.org/wiki/Device_mapper) -* [https://docs.kernel.org/admin-guide/device-mapper](https://docs.kernel.org/admin-guide/device-mapper) +* [https://docs.kernel.org/admin-guide/device-mapper/index.html](https://docs.kernel.org/admin-guide/device-mapper/index.html) Device mapper is a kernel framework for creating virtual block devices from one or more underlying devices. Technologies such as LVM, @@ -244,11 +276,11 @@ later pool if desired. across many independent, already-formatted filesystems. -## UnRAID +## Unraid * [https://unraid.net](https://unraid.net) -UnRAID is a full OS platform that offers pooled storage with integrated +Unraid is a full OS platform that offers pooled storage with integrated real-time parity. Compared with mergerfs, its main advantage is that it bundles pooling and parity in one product, while mergerfs is a standalone Linux pooling filesystem without built-in redundancy. For @@ -257,8 +289,8 @@ approach is also preferable. For semi-static data, mergerfs + [SnapRAID](https://www.snapraid.it) provides a similar, but not real-time, solution. -[NonRAID](https://github.com/qvr/nonraid) (see below) is a -fork of UnRAID's parity calculation solution and can also be used with +[NonRAID](https://github.com/qvr/nonraid) (see [below](#nonraid)) is a +fork of Unraid's parity calculation solution and can also be used with mergerfs. * **Better than mergerfs for:** turnkey pooling plus integrated real-time @@ -272,7 +304,7 @@ mergerfs. * [https://github.com/qvr/nonraid](https://github.com/qvr/nonraid) [NonRAID](https://github.com/qvr/nonraid#how-it-works) is a fork of -UnRAID's storage array kernel driver. Effectively it is like RAID5 at +Unraid's storage array kernel driver. Effectively it is like RAID5 at the block device layer without treating the collection of devices as a single device. Each device gets a virtual block device associated with it which can be formatted with the filesystem of the user's choice and @@ -293,7 +325,7 @@ be used with mergerfs for those looking for a unified view. ## ZFS -* [https://en.wikipedia.org/wiki/ZFS](https://en.wikipedia.org/wiki/ZFS) +* [https://openzfs.github.io/openzfs-docs](https://openzfs.github.io/openzfs-docs) ZFS is an advanced filesystem that combines storage pooling, snapshots, data integrity checking, and built-in RAID-like redundancy @@ -313,72 +345,78 @@ without built-in redundancy. ## ZFS AnyRAID -* [https://github.com/openzfs/zfs/discussions/16773](https://github.com/openzfs/zfs/discussions/16773) +* [Announcement](https://docs.hexos.com/blog/2025-05-22) +* [Development update](https://docs.hexos.com/blog/2026-7-15) -[ZFS AnyRAID](https://github.com/openzfs/zfs/discussions/16773) -([announcement](https://hexos.com/blog/introducing-zfs-anyraid-sponsored-by-eshtek)) -is a feature under development for ZFS intended to provide more -flexibility in pools by allowing mixed-capacity disks while -maintaining live redundancy. +ZFS AnyRAID is under development to support mixed-capacity disks while +retaining ZFS redundancy. The planned layouts include mirrored and RAID-Z +variants, along with rebalancing and pool contraction work. -If released, ZFS AnyRAID would offer some of the flexibility associated -with mergerfs-style mixed-capacity expansion, but with built-in -redundancy inside ZFS. mergerfs already provides that mixed-size, -add-drives-anytime flexibility today, albeit without built-in -redundancy. If you need redundancy with that flexibility, ZFS AnyRAID -could become an option when available; until then, mergerfs remains the -simpler currently available choice for mixed-capacity pooling, with -redundancy and integrity available via SnapRAID and/or NonRAID. +Availability is still landing in OpenZFS. Published notes indicate initial +mirror layouts may ship in a 2.4.x release, with broader RAID-Z variants, +rebalancing, and pool contraction targeted for later releases. Treat +release and downstream product-integration dates as roadmap targets +rather than commitments. + +Where available, AnyRAID would offer some of the mixed-size flexibility +associated with mergerfs while retaining redundancy inside ZFS. mergerfs +provides mixed-size, add-drives-anytime pooling today, but without +built-in redundancy; SnapRAID or NonRAID can provide separate +protection. * **Better than mergerfs for:** users who want mixed-capacity flexibility and built-in live redundancy in one filesystem stack. -* **Worse than mergerfs for:** users who need mature availability now and - the simplest add-drives-anytime workflow without redundancy overhead. +* **Worse than mergerfs for:** users who need production availability now or + want to pool existing filesystems without migrating data. -## Bcachefs +## bcachefs * [https://bcachefs.org](https://bcachefs.org) -Bcachefs is a modern copy-on-write filesystem for Linux designed for -multi-device storage aggregation. It includes integrated features such -as checksumming, snapshots, compression, encryption, and redundancy. +bcachefs is a modern copy-on-write filesystem for Linux designed for +multi-device storage aggregation. It includes integrated features such as +checksumming, snapshots, compression, encryption, and redundancy. -Compared with mergerfs, bcachefs is a full integrated filesystem with -built-in checksumming, snapshots, compression, and redundancy, while -mergerfs is a lightweight pooling layer over existing filesystems. -Bcachefs is still evolving and should be evaluated carefully for -production use. +Compared with mergerfs, bcachefs is a full integrated filesystem, while +mergerfs is a lightweight pooling layer over existing filesystems. Beginning +with Linux 6.18, bcachefs is no longer distributed in the mainline kernel; the +project now distributes it as an external DKMS module. It remains actively +developed, but deployments must account for kernel compatibility and its +shorter operational history. * **Better than mergerfs for:** integrated CoW filesystem features, checksumming, snapshots, compression, and built-in multi-device redundancy. -* **Worse than mergerfs for:** conservative production setups prioritizing - maturity, simplicity, and broad long-term deployment stability. +* **Worse than mergerfs for:** pooling existing filesystems without + reformatting, mainline-kernel availability, and per-disk native access. -## Btrfs Single Data Profile +## Btrfs multi-device filesystems -* [https://btrfs.readthedocs.io](https://btrfs.readthedocs.io) +* [https://btrfs.readthedocs.io/en/latest/Volume-management.html](https://btrfs.readthedocs.io/en/latest/Volume-management.html) -[Btrfs'](https://btrfs.readthedocs.io) `single` data profile is a -multi-device Btrfs mode that can spread data across drives without -adding redundancy. +Btrfs can combine multiple devices in one filesystem and select separate +allocation profiles for data and metadata. The `single` data profile spreads +one copy of data across available devices without redundancy; other profiles +provide mirroring, striping, or parity with different capacity and failure +trade-offs. -Compared with mergerfs, Btrfs `single` is managed as one integrated -filesystem, while mergerfs pools separate existing filesystems. In -Btrfs `single`, losing one device can impact the whole filesystem; -with mergerfs, loss is typically limited to data on the failed disk. +Compared with mergerfs, multi-device Btrfs is one integrated filesystem, +while mergerfs pools separate existing filesystems. Losing a device from a +`single` profile can make the whole Btrfs filesystem incomplete. Redundant +profiles change that risk, but the member devices still are not independently +mountable filesystems. -* **Better than mergerfs for:** integrated multi-device management in one - filesystem namespace. -* **Worse than mergerfs for:** users who value per-disk failure isolation - and easy independent recovery. +* **Better than mergerfs for:** integrated checksumming, snapshots, balancing, + and selectable redundancy profiles. +* **Worse than mergerfs for:** pooling existing mixed filesystem types, + per-disk native access, and failure isolation without redundancy. ## StableBit's DrivePool -* [https://stablebit.com](https://stablebit.com) +* [https://stablebit.com/DrivePool/Features](https://stablebit.com/DrivePool/Features) DrivePool is a commercial storage pooling product for Windows that combines multiple drives into a single volume and includes balancing @@ -399,19 +437,17 @@ external-tool composability. * [https://learn.microsoft.com/en-us/windows-server/storage/storage-spaces/overview](https://learn.microsoft.com/en-us/windows-server/storage/storage-spaces/overview) -Windows Storage Spaces is Microsoft's software-defined storage -platform for Windows Server and Windows Pro. It aggregates multiple -physical disks into logical storage pools with built-in redundancy -options (mirroring, parity) and automatic performance optimization -through tiered caching. +Windows Storage Spaces is Microsoft's block-level storage virtualization +platform for Windows. It aggregates physical disks into storage pools from +which virtual disks can be created with simple, mirror, or parity layouts. +Supported Windows Server configurations can also use faster media as a +Storage Bus Cache. -Storage Spaces is Windows-only and requires more planning upfront due -to redundancy options and RAID-like features. mergerfs is -Linux-focused and takes a simpler approach, pooling existing -filesystems without built-in redundancy. If you need a Windows -solution with integrated data protection, Storage Spaces is a good -choice. For Linux users seeking lightweight pooling without redundancy -overhead, mergerfs is the better option. +Unlike mergerfs, Storage Spaces manages the member devices below the +filesystem and stripes or mirrors data according to the selected layout. +This can provide integrated resilience and one contiguous volume, but member +disks are not independently readable filesystems. mergerfs instead pools +existing filesystems and leaves their files directly accessible. * **Better than mergerfs for:** Windows environments needing integrated pooling, resilience, and tiering in the platform stack. @@ -439,6 +475,25 @@ Plan 9 isn't a widely used OS so this comparison is mostly academic. practical deployment availability. +## 9P + +* [https://en.wikipedia.org/wiki/9P_(protocol)](https://en.wikipedia.org/wiki/9P_(protocol)) + +[9P](https://en.wikipedia.org/wiki/9P_(protocol)) is a filesystem +protocol from the Plan 9 operating system used to access files over a +network between systems. + +Compared with mergerfs, 9P is a network filesystem protocol, while +mergerfs is a local Linux FUSE filesystem for pooling directories and +filesystems under one mount with branch policies. Like the Plan 9 binds +comparison above, this is mostly academic for typical mergerfs users. + +* **Better than mergerfs for:** lightweight network filesystem protocol + use between compatible systems. +* **Worse than mergerfs for:** local Linux filesystem pooling because it + does not provide mergerfs-like branch policies or pooling semantics. + + ## SnapRAID pooling * [https://www.snapraid.it/manual](https://www.snapraid.it/manual) @@ -498,7 +553,7 @@ given the differing feature sets and focuses of the two projects. ## distributed filesystems * [AFS](https://www.openafs.org) -* [Ceph/CephFS](https://ceph.io) +* [Ceph/CephFS](https://docs.ceph.com/en/latest/cephfs/) * [GlusterFS](https://www.gluster.org) * [LizardFS](https://lizardfs.com) * [MooseFS](https://moosefs.com) @@ -539,8 +594,8 @@ pooling through normal POSIX filesystem calls. Given its design nofs is not suited for general usage as third-party applications will not be able to take advantage of the unioning -behavior it offers. It is primarily for more simple situations where -something like mergerfs is unable to be used. +behavior it offers. It is primarily for simpler situations where +mergerfs cannot be used. * **Better than mergerfs for:** constrained environments where a FUSE mount is not possible but command-style union behavior is acceptable. @@ -548,32 +603,34 @@ something like mergerfs is unable to be used. usable by arbitrary third-party applications. -## policyfs +## PolicyFS * [https://policyfs.org](https://policyfs.org) * [https://github.com/hieutdo/policyfs](https://github.com/hieutdo/policyfs) -policyfs is a Linux FUSE storage daemon that unifies multiple storage +PolicyFS is a Linux FUSE storage daemon that unifies multiple storage paths under a single mountpoint, similarly to mergerfs. Its distinguishing features are explicit path-pattern-based routing rules for placement, and an optional SQLite metadata index that can serve `readdir` and `getattr` operations without spinning up HDDs. -Compared with mergerfs, policyfs emphasizes explicit path-based routing -and optional metadata indexing, while mergerfs emphasizes a broad, -policy-driven pooling model with longer operational history and wider -adoption. It doesn't try limiting spinup as doing so fundimentally is -in [conflict with other features.](faq/limit_drive_spinup.md) +Compared with mergerfs, PolicyFS emphasizes explicit path-based routing, +scheduled SSD-to-HDD movement, and optional metadata indexing. mergerfs +emphasizes a broader policy-driven pooling model with longer operational +history and wider application compatibility. mergerfs does not try to prevent +metadata operations from waking branches because doing so fundamentally +[conflicts with other features.](faq/limit_drive_spinup.md) -Due to the nature of PolicyFS it can not really be used while changing -files out-of-band and with the delayed writeback of metadata and -unlinking there is a greater risk of inconsistancy in error -conditions. +PolicyFS's indexed storage deliberately defers some delete, rename, and +attribute operations until a maintenance job runs. Out-of-band changes require +the index to be refreshed, and its documentation identifies limits around +`O_DIRECT`, advisory locks, NFS re-export, and full POSIX coverage. Those are +material trade-offs, not only maturity differences. -* **Better than mergerfs for:** explicit path-based routing and optional - metadata indexing to reduce HDD spin-ups. -* **Worse than mergerfs for:** users prioritizing project maturity, - adoption, performance, and long-established operational behavior. +* **Better than mergerfs for:** explicit path-based routing, scheduled + tiering, and metadata indexing designed to reduce HDD spin-ups. +* **Worse than mergerfs for:** broad POSIX compatibility, immediate mutation + semantics on every branch, and long-established operational behavior. ## Greyhole @@ -593,22 +650,4 @@ Linux FUSE mount for local pooling via standard POSIX filesystem calls. * **Better than mergerfs for:** Samba-centric environments that need built-in per-share redundancy in the same tool. * **Worse than mergerfs for:** general-purpose local filesystem pooling - outside Samba and heavy small-file churn workloads. - - -## 9P - -* [https://en.wikipedia.org/wiki/9P_(protocol)](https://en.wikipedia.org/wiki/9P_(protocol)) - -[9P](https://en.wikipedia.org/wiki/9P_(protocol)) is a filesystem -protocol from the Plan 9 operating system used to access files over a -network between systems. - -Compared with mergerfs, 9P is a network filesystem protocol, while -mergerfs is a local Linux FUSE filesystem for pooling directories and -filesystems under one mount with branch policies. - -* **Better than mergerfs for:** lightweight network filesystem protocol - use between compatible systems. -* **Worse than mergerfs for:** local Linux filesystem pooling because it - does not provide mergerfs-like branch policies or pooling semantics. + outside Samba, and for heavy small-file churn workloads. diff --git a/mkdocs/docs/related_projects.md b/mkdocs/docs/related_projects.md index f43bde1c..f40bb768 100644 --- a/mkdocs/docs/related_projects.md +++ b/mkdocs/docs/related_projects.md @@ -27,9 +27,9 @@ disk arrays, storing parity information for data recovery in the event of up to six disk failures. * [nonraid](https://github.com/qvr/nonraid): NonRAID is a fork of the - unRAID system's open-source md_unraid kernel driver enabling - UnRAID-style storage arrays with parity protection outside of the - commercial UnRAID system. + Unraid system's open-source md_unraid kernel driver enabling + Unraid-style storage arrays with parity protection outside of the + commercial Unraid system. * [rclone](https://rclone.org): a command-line program to manage files on cloud storage. It is a feature-rich alternative to cloud vendors' web storage interfaces. rclone's @@ -43,10 +43,10 @@ * [Proxmox](https://www.proxmox.com): Proxmox is a popular, Debian based, virtualization platform. Users tend to install mergerfs on the host and pass the mount into containers. -* [UnRAID](https://unraid.net): "Unraid is a powerful, easy-to-use +* [Unraid](https://unraid.net): "Unraid is a powerful, easy-to-use operating system for self-hosted servers and network-attached - storage." While UnRAID has its own union filesystem it isn't - uncommon to see UnRAID users leverage mergerfs given the differences + storage." While Unraid has its own union filesystem it isn't + uncommon to see Unraid users leverage mergerfs given the differences in the technologies. There is a [plugin available by Rysz](https://forums.unraid.net/topic/144999-plugin-mergerfs-for-unraid-support-topic) to ease installation and setup. From fcd7527c9d2b71ad49b169650e5a1cff38519226 Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Thu, 1 Oct 2026 18:38:46 -0500 Subject: [PATCH 22/23] docs: document mergerfs tradeoffs and web UI management * Add: a comprehensive downsides FAQ with mitigations and practical priorities. * Document: mergerfs-webui and link it from setup and configuration guides. * Replace: obsolete mergerfs.fsck references with fsck.mergerfs guidance. --- mkdocs/docs/faq/configuration_and_policies.md | 6 + mkdocs/docs/faq/downsides.md | 218 ++++++++++++++++++ mkdocs/docs/faq/why_isnt_it_working.md | 2 +- mkdocs/docs/quickstart.md | 11 + mkdocs/docs/related_projects.md | 3 + mkdocs/docs/runtime_interface.md | 5 + mkdocs/docs/tips_notes.md | 3 +- mkdocs/docs/tooling.md | 25 +- mkdocs/mkdocs.yml | 1 + 9 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 mkdocs/docs/faq/downsides.md diff --git a/mkdocs/docs/faq/configuration_and_policies.md b/mkdocs/docs/faq/configuration_and_policies.md index 6bfba4d1..c7e6156c 100644 --- a/mkdocs/docs/faq/configuration_and_policies.md +++ b/mkdocs/docs/faq/configuration_and_policies.md @@ -18,6 +18,12 @@ mergerfs [does not impact](usage_and_functionality.md) the underlying filesystems and can be added or removed without any impact it is extremely easy to test and experiment with different settings. +[mergerfs-webui](../tooling.md#mergerfs-webui) provides a browser-based +way to inspect and change settings, branches, and policies on Linux, +and edit startup configuration. It does not replace understanding +what the options do or choosing policies suited to your use case. +Runtime changes alone do not persist across remounts. + Additional reading: * [Config/Options](../config/options.md) diff --git a/mkdocs/docs/faq/downsides.md b/mkdocs/docs/faq/downsides.md new file mode 100644 index 00000000..8233d52f --- /dev/null +++ b/mkdocs/docs/faq/downsides.md @@ -0,0 +1,218 @@ +# Downsides + +## What are the downsides of using mergerfs? + +mergerfs trades the features of an integrated storage system for the +flexibility of pooling independent filesystems. It is a filesystem +proxy, not RAID or a replacement for the underlying filesystems. + +Some limitations are inherent to that design; others matter only for +particular workloads, configurations, or applications. Many can be +mitigated with appropriate settings or separate tools. The following +is a checklist of tradeoffs to evaluate, not a list of problems every +installation will experience. + +### No built-in data protection or storage management + +mergerfs does not provide redundancy, parity, integrity checks, +snapshots, or file versioning. If a drive fails, mergerfs cannot +recover the files on it. The other drives retain their independent +filesystems, but that failure isolation is not redundancy or a backup. +Protection must come from the underlying storage or separate tools, +such as [SnapRAID](https://www.snapraid.it) or +[NonRAID](https://github.com/qvr/nonraid). + +It also does not automatically rebalance existing files when a branch +is added or space usage changes. If redistribution is needed, it must +be done separately. + +See [Non-features](../index.md#non-features), [Project +Comparisons](../project_comparisons.md), and [How can I balance files +across the pool?](configuration_and_policies.md#how-can-i-balance-files-across-the-pool). + +### No striping or contiguous pooled space + +Each file lives entirely on one branch. Adding drives does not make a +single-file transfer faster, and a file must fit on one eligible +branch even if the pool reports enough total free space. For example, +two filesystems with 6GiB free each cannot hold one 10GiB file. +Concurrent access to different branches can still provide aggregate +throughput. + +See [Project Comparisons](../project_comparisons.md#raid0-jbod-span-drive-concatenation-striping) +and [Performance](../performance.md#understanding-mergerfs-io-performance). + +### Performance overhead + +Filesystem requests generally pass through FUSE and the mergerfs +process before reaching the underlying filesystem. This adds latency +and CPU overhead compared to direct access. Directory listings and +other metadata operations can require work across multiple branches, +so their cost can grow as the pool grows. Page caching can also result +in data being cached both through mergerfs and on the underlying +filesystem. + +[IO passthrough](../config/passthrough.md) can reduce data IO overhead, +but does not eliminate the work of combining branches. Databases, VM +images, and other latency-sensitive workloads should be benchmarked +with the intended configuration rather than assumed to perform like +direct access. + +There is also memory overhead beyond page caching: request buffers, +temporary directory buffers, and FUSE node bookkeeping. Usage depends +on concurrency, buffer sizes, and the workload. Exporting the pool +over NFS requires retaining nodes with `never-forget-nodes=true`, +which can increase memory usage as more entries are accessed. This is +worth considering on memory-constrained systems or large NFS exports. +See [Resource Usage](../resource_usage.md) and [NFS exporting +mergerfs](../remote_filesystems.md#nfs-exporting-mergerfs). + +See [Understanding mergerfs IO +Performance](../performance.md#understanding-mergerfs-io-performance), +[Benchmarking](../benchmarking.md), and [Why use +FUSE?](technical_behavior_and_limitations.md#why-use-fuse-why-not-a-kernel-based-solution). + +### Placement and filesystem semantics need attention + +A single mount point does not make the branches one filesystem. +[Policies](../config/functions_categories_policies.md) determine where +new files go and which copy is used when the same path exists on +multiple branches. Path-preserving policies and +[minfreespace](../config/minfreespace.md) can exclude branches from file +creation even while the pool has free space. mergerfs cannot know a +file's final size when it is created. + +Hard links and renames still have underlying filesystem constraints. +Depending on the layout and policies, an operation can return `EXDEV` +(cross-device link) even within the pool. Applications which do not +handle that error may fail rather than fall back to copying. + +Directory metadata is also selected by policy rather than necessarily +aggregated. With the default `getattr` policy of `ff`, a directory's +timestamp comes from the first matching branch. A media scanner which +uses that timestamp to skip unchanged directories can miss updates on +another branch. `func.getattr=newest` can help with timestamp-based +scans, but changes which branch supplies metadata; it does not change +which file other operations select. + +Corresponding directories on different branches can have different +ownership or permissions. Since access uses the caller's credentials, +this can result in unexpected access errors or missing entries. +Keeping those permissions consistent and auditing with +`fsck.mergerfs` helps. Container UID/GID mappings and non-POSIX +branches may need additional attention; mergerfs does not bypass +their access restrictions. + +See [Configuration and Policies](configuration_and_policies.md), +[rename and link](../config/rename_and_link.md), [Tips and +Notes](../tips_notes.md), and [How does mergerfs interact with user +namespaces?](compatibility_and_integration.md#how-does-mergerfs-interact-with-user-namespaces). + +### Keeping drives spun down can be harder + +Directory listings and file searches can query multiple branches, +potentially waking drives beyond the one containing the requested +file. If keeping disks asleep is important, this can increase power +usage, noise, and access latency. It is not a concern for SSDs or +drives which are kept spinning. + +Caching and reducing background scans can limit some branch access, +but mergerfs alone cannot guarantee that unused disks stay asleep. +See [Limiting drive spinup](limit_drive_spinup.md) for the tradeoffs +and possible approaches. + +### Direct branch changes and cached views + +Using the underlying filesystems directly is supported, but changes +made outside the mergerfs mount are not forwarded as `inotify` or +`fanotify` events to applications watching the pool. Changes made +through the pool do generate notifications. If files are added +directly to branches, applications such as media servers may need +periodic scans, explicit refreshes, or watches on the branches +themselves. + +Metadata caching introduces a separate visibility tradeoff: direct +branch changes may not appear through the pool until cached entries +expire. Longer cache durations can reduce filesystem queries at the +cost of less immediate visibility. Shorter durations or disabling the +relevant caches can help if direct branch changes are common. Avoid +uncoordinated writes to the same file through both paths, especially +when file caching is enabled. + +See [Does inotify and fanotify +work?](compatibility_and_integration.md#does-inotify-and-fanotify-work), +[Can filesystems still be used +directly?](usage_and_functionality.md#can-filesystems-still-be-used-directly-outside-of-mergerfs-while-pooled), +and [Caching](../config/cache.md). + +### Branch paths do not guarantee a particular disk is mounted + +mergerfs works on paths and does not require each branch to be a +mount point. If an intended branch filesystem is not mounted but its +mount-point directory still exists, mergerfs can use that directory +on the filesystem beneath it. If writable and selected for creation, +new files can land there instead of on the intended disk, potentially +filling the root filesystem. + +This is a deployment concern when pooling mounted filesystems, not a +problem with intentionally using ordinary directories as branches. +Configure mount dependencies or +`branches-mount-timeout` with `branches-mount-timeout-fail=true` if +startup should fail when branches are not ready. These are startup +safeguards, not continuous checks that the intended disks remain +mounted at runtime. + +See [branches-mount-timeout](../config/branches-mount-timeout.md) and +[What happens if a filesystem disappears at +runtime?](usage_and_functionality.md#what-happens-if-a-filesystem-disappears-at-runtime). + +### Dependence on branch responsiveness + +mergerfs cannot make an unresponsive disk or remote filesystem +responsive. A request to a blocked branch blocks the thread handling +it. If enough threads are blocked, the pool can stop responding as +well. Independent on-disk filesystems do not guarantee that a failing +branch has no effect on access through the pool. + +See [What happens if a branch filesystem +blocks?](technical_behavior_and_limitations.md#what-happens-if-a-branch-filesystem-blocks). + +### Compatibility limitations + +Some filesystem features are unavailable through FUSE or require +particular kernel versions and settings. For example, reflink requests +(`FICLONE` / `FICLONERANGE`) are not supported through the mergerfs +mount. Applications using `mmap` need page caching; older Linux or +mergerfs versions require it to be explicitly enabled. + +Advisory locks taken through the pool are not forwarded to the +underlying branches. Local applications using only the mergerfs mount +still have kernel-managed locks, but those locks do not coordinate +with direct branch access or other hosts accessing a remote branch. + +Inode identity also has tradeoffs when presenting multiple filesystems +as one. Most applications do not need special settings, but backup or +deduplication tools and NFS can be sensitive to inode values changing +when files move between branches or a policy selects a different +copy. [inodecalc](../config/inodecalc.md) offers different strategies: +for example, `path-hash` stabilizes identity by path for NFS, but +prevents applications from recognizing hard links by inode equality. +The hard links themselves still work. + +On SELinux systems, per-file relabeling is unavailable through the +FUSE mount, so container bind-mount options `:z` and `:Z` do not +perform their usual relabeling. This does not mean containers cannot +use the pool: an appropriate SELinux policy or a mount-wide context +can allow access. A mount-wide context is not a substitute for +distinct per-file labels if the application requires them. See [Does +mergerfs work with SELinux +relabeling?](compatibility_and_integration.md#does-mergerfs-work-with-selinux-relabeling). + +See [Technical Behavior and +Limitations](technical_behavior_and_limitations.md) and [Known Issues +and Bugs](../known_issues_bugs.md). + +These tradeoffs do not require migrating data into a special format. +The underlying filesystems remain usable without mergerfs, and +mergerfs can be [removed without changing their +data](usage_and_functionality.md#can-mergerfs-be-removed-without-affecting-the-data). diff --git a/mkdocs/docs/faq/why_isnt_it_working.md b/mkdocs/docs/faq/why_isnt_it_working.md index bc4f46b8..0d1560bf 100644 --- a/mkdocs/docs/faq/why_isnt_it_working.md +++ b/mkdocs/docs/faq/why_isnt_it_working.md @@ -119,7 +119,7 @@ may be able to read some files and directories on one filesystem but not another resulting in an incomplete set. And if one of the branches it can access is empty then it will return an empty list. -Try using [mergerfs.fsck](https://github.com/trapexit/mergerfs-tools) +Try using [fsck.mergerfs](../tooling.md#fsckmergerfs) tool to check for and fix inconsistencies in permissions. If you aren't seeing anything at all be sure that the basic permissions are correct. The user and group values are correct and that directories diff --git a/mkdocs/docs/quickstart.md b/mkdocs/docs/quickstart.md index 374d97c8..a406795c 100644 --- a/mkdocs/docs/quickstart.md +++ b/mkdocs/docs/quickstart.md @@ -62,6 +62,17 @@ For some suggestions on branch setup see [the page on branches](config/branches.md#branch-setup). +### Web UI + +[mergerfs-webui](tooling.md#mergerfs-webui) provides a browser-based +alternative for creating and managing mounts on Linux. It can edit +startup configuration and inspect and change settings on running +mounts. See the +[mergerfs-webui documentation](https://github.com/trapexit/mergerfs-webui#readme) +for installation and secure access guidance. Runtime changes alone do +not persist across remounts. + + ### Command Line ``` diff --git a/mkdocs/docs/related_projects.md b/mkdocs/docs/related_projects.md index f40bb768..139e8cda 100644 --- a/mkdocs/docs/related_projects.md +++ b/mkdocs/docs/related_projects.md @@ -23,6 +23,9 @@ ## Software and services commonly used with mergerfs +* [mergerfs-webui](tooling.md#mergerfs-webui): a browser-based tool for + managing mergerfs mounts, runtime settings, and startup configuration + on Linux. * [snapraid](https://www.snapraid.it): a backup program designed for disk arrays, storing parity information for data recovery in the event of up to six disk failures. diff --git a/mkdocs/docs/runtime_interface.md b/mkdocs/docs/runtime_interface.md index 3230cbf4..8e1bf9df 100644 --- a/mkdocs/docs/runtime_interface.md +++ b/mkdocs/docs/runtime_interface.md @@ -14,6 +14,11 @@ getting and setting config and issuing certain commands. The other are files found on the filesystem for querying certain `mergerfs` specific information about them. +For a browser-based interface to inspect and change runtime settings, +manage branches and policies, and issue commands, see +[mergerfs-webui](tooling.md#mergerfs-webui). Runtime changes still need +to be saved in startup configuration if they should persist. + ## .mergerfs pseudo file diff --git a/mkdocs/docs/tips_notes.md b/mkdocs/docs/tips_notes.md index 289f8a61..3faf5509 100644 --- a/mkdocs/docs/tips_notes.md +++ b/mkdocs/docs/tips_notes.md @@ -14,7 +14,8 @@ * If you do not see some directories and files you expect, policies seem to skip branches, you get strange permission errors, etc. be sure the underlying filesystems' permissions are all the same. Use - `mergerfs.fsck` to audit the filesystem for out of sync permissions. + [fsck.mergerfs](tooling.md#fsckmergerfs) to audit the filesystem for + out of sync permissions. * If you still have permission issues be sure you are using POSIX compliant filesystems. mergerfs doesn't generally make exceptions for FAT, NTFS, or other non-POSIX filesystem. diff --git a/mkdocs/docs/tooling.md b/mkdocs/docs/tooling.md index f83a0c8c..0f5d0e93 100644 --- a/mkdocs/docs/tooling.md +++ b/mkdocs/docs/tooling.md @@ -1,5 +1,28 @@ # Tooling +## mergerfs-webui + +[mergerfs-webui](https://github.com/trapexit/mergerfs-webui) is a +browser-based tool for viewing and managing mergerfs mounts on Linux. +It is a separate project with a single executable containing both the +web server and the web page. + +It can: + +* Install mergerfs and create mounts through `/etc/fstab` or systemd + `.mount` units. +* Discover running mounts, mount and unmount them, and inspect and + change their runtime settings, branches, and policies. +* Edit startup configuration in `/etc/fstab`, systemd `.mount` units, + and referenced mergerfs ini files, with previews of structured + changes before saving. +* Issue runtime commands and set up the web UI as a systemd service. + +[Runtime changes](runtime_interface.md) are not automatically +persistent; startup configuration must also be updated if changes +should survive a remount. + + ## mergerfs.collect-info A tool included in recent releases of `mergerfs` which collects @@ -161,8 +184,6 @@ Environment=LD_PRELOAD=/usr/lib/mergerfs/preload.so behaviors that can be build on top of mergerfs. They may not have all the features you are looking for.** * mergerfs.ctl: A tool to make it easier to query and configure mergerfs at runtime - * mergerfs.fsck: Provides permissions and ownership auditing and - the ability to fix them (should use `fsck.mergerfs` instead) * mergerfs.dedup: Will help identify and optionally remove duplicate files * mergerfs.dup: Ensure there are at least N copies of a file across the pool * mergerfs.balance: Rebalance files across filesystems by moving them from the most filled to the least filled diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index a8fc518c..82242318 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -115,6 +115,7 @@ nav: - faq/compatibility_and_integration.md - faq/technical_behavior_and_limitations.md - faq/have_you_considered.md + - faq/downsides.md - faq/limit_drive_spinup.md - Related Projects: related_projects.md - Media and Publicity: media_and_publicity.md From 8e87bcbff2689872e61752737308570717af1a3a Mon Sep 17 00:00:00 2001 From: Antonio SJ Musumeci Date: Fri, 2 Oct 2026 13:57:01 -0500 Subject: [PATCH 23/23] docs: Add Unraid migration guidance for mergerfs - Add a top-level Migration page covering both migration directions. - Explain filesystem reuse, parity constraints, partition requirements, and unlocking encrypted Unraid disks. - Describe intact ZFS pool imports and subdirectory branch reuse, distinguishing them from cases requiring a data copy. - Add navigation and link the guide from the Unraid comparison. --- mkdocs/docs/migration.md | 185 +++++++++++++++++++++++++++++ mkdocs/docs/project_comparisons.md | 10 +- mkdocs/mkdocs.yml | 1 + 3 files changed, 194 insertions(+), 2 deletions(-) create mode 100644 mkdocs/docs/migration.md diff --git a/mkdocs/docs/migration.md b/mkdocs/docs/migration.md new file mode 100644 index 00000000..d9c525b5 --- /dev/null +++ b/mkdocs/docs/migration.md @@ -0,0 +1,185 @@ +# Migration + +## Unraid + +mergerfs and a parity-protected Unraid array can both present +directories from multiple filesystems as a single view, but they treat +storage membership very differently. With mergerfs, membership is a +property of its **configuration**: branches are paths and can be added +or removed at runtime without modifying the underlying filesystems. +With Unraid's main array, membership is part of the **array state**: +adding a data disk to an array with active parity normally requires +clearing and formatting it, and removing one normally requires parity +to be rebuilt. + +Those array-level requirements, not the User Share filesystem itself, +are what shape migrations in either direction. When moving to mergerfs, +existing data filesystems can usually be reused as-is. When moving to +Unraid, populated filesystems can normally only be brought into the +array before parity is activated. See [Project +Comparisons](project_comparisons.md#unraid) for the comparison itself. + +| Operation | mergerfs | Unraid array with parity | +| --- | --- | --- | +| Add an empty filesystem | Add its mount path as a branch | Clear if necessary, then format and add | +| Add a populated filesystem | Add its mount path; data is immediately visible | Not normally possible once parity is active without destroying its contents | +| Mix filesystem types | Any filesystem the host can mount | Supported array filesystem types only | +| Mix device sizes | Arbitrary | Data disks cannot exceed the parity disk size | +| Change membership while mounted | Supported through the [runtime interface](runtime_interface.md) | Array configuration normally requires stopping and restarting the array | +| Remove a filesystem and retain it unchanged | Remove the branch | Possible, but normal array removal requires parity to be rebuilt | +| Remove a filesystem without recalculating parity | Not applicable | Requires zeroing the disk, destroying its filesystem | + +### Migrating from Unraid to mergerfs + +Unraid array data disks each hold a complete, individually formatted +filesystem. `shfs` only unions their top-level directories, so there +is no striped or parity-dependent layout to unwind. After cleanly +stopping the array, unencrypted data filesystems can be mounted under +Linux and their mount points used directly as mergerfs branches. The +files do not need to be rewritten merely because the pooling +implementation changed. + +For [encrypted Unraid disks](https://docs.unraid.net/unraid-os/system-administration/secure-your-server/securing-your-data/), +securely retain the existing passphrase or key file before retiring +Unraid. Open each disk's existing LUKS container with `cryptsetup` +using those credentials, then mount the inner filesystem through its +`/dev/mapper/` device and use that mount point as a branch. Do not +reformat the disk or create a new LUKS container. Without a working +passphrase or key file, the encrypted data cannot be accessed. + +``` +Unraid: + + disk1 ─┐ + disk2 ─┼─> shfs ─> /mnt/user + disk3 ─┘ + + + parity + +mergerfs: + + disk1 ─┐ + disk2 ─┼─> mergerfs ─> /mnt/storage + disk3 ─┘ +``` + +Unraid's convention of representing shares as identically named +top-level directories also maps naturally to mergerfs' union +semantics. + +* The parity disk is not a data filesystem and cannot become a branch + as-is. It can be reformatted as another data disk if parity + protection is no longer desired, or a separate parity solution such + as [SnapRAID](https://www.snapraid.it) or + [NonRAID](https://github.com/qvr/nonraid) can be used alongside + mergerfs if redundancy is still wanted. +* If migrating incrementally while keeping the remaining Unraid array + operational, the normal Unraid disk-removal and parity-rebuild + requirements still apply to each disk removed. +* If the entire array is being retired at once, those intermediate + Unraid parity rebuilds are unnecessary. Stop the array cleanly and + use the existing data filesystems as branches. +* Named pools need care. A pool backed by a single filesystem can be + mounted and used as one branch. A multi-device Btrfs or ZFS pool is + not equivalent to a collection of independent Unraid array disks: it + can either remain intact and be mounted as one mergerfs branch, or + its data must be migrated elsewhere before its component devices can + be repurposed as independent filesystems. + +### Migrating from mergerfs to Unraid + +mergerfs places no special metadata on its branches, so their +underlying filesystems remain unchanged. Direct reuse as Unraid array +data disks requires local disks with a supported filesystem **and a +partition layout accepted by the installed Unraid version**. A +supported filesystem type alone is not sufficient. + +In particular, mergerfs can use a filesystem formatted directly on a +whole device, such as `/dev/sda`, without a partition table. Unraid +array data disks require a filesystem within a compatible partition. +Such unpartitioned filesystems cannot be imported directly: back up +their data, let Unraid partition and format the disks, then restore +the files. Creating a partition table over the existing filesystem is +not a safe conversion. Check Unraid's +[partitioning guidance](https://docs.unraid.net/unraid-os/troubleshooting/faq/) +before planning a no-copy migration. + +Current versions of Unraid support importing populated disks when +constructing a new array, or after resetting the array configuration, +provided that the populated data disks are added **before parity is +activated**. Such a migration can therefore often be performed without +copying the data: + +1. Stop mergerfs and cleanly unmount the branch filesystems. +2. Create or reset the Unraid array configuration. +3. Assign all populated mergerfs disks as data disks while no parity + disk is active. +4. Verify that the existing filesystems and data are recognized + correctly. If a populated disk is reported as unmountable, stop and + investigate; do not format it, as formatting destroys its contents. +5. Assign a parity disk at least as large as the largest data disk. +6. Build parity over the completed array. + +Once parity has been established, that convenient path is effectively +closed. Adding another populated mergerfs disk to the parity-protected +array would require Unraid to clear the disk, destroying the existing +filesystem. A staged migration in which populated mergerfs disks are +added one at a time therefore generally requires either copying their +contents into the existing array first or resetting/reconstructing the +Unraid configuration so that all populated data disks are present +before parity is established. + +Additional array-level constraints to keep in mind: + +* Only filesystem types supported by Unraid can be assigned as array + data disks. +* No array data disk may be larger than the parity disk. +* Array configuration changes normally require stopping and restarting + the array. Removing a data disk normally requires a parity rebuild, + and Unraid's parity-preserving removal procedure works by erasing + the filesystem and zeroing the disk, so it is unsuitable when the + goal is to retain that filesystem for use elsewhere. + +When migrating into individual **Unraid array data slots**, the +following configurations require copying data to suitable array +filesystems rather than reusing the source storage directly: + +* Remote filesystems or filesystem types Unraid does not support. +* Storage that cannot be presented as a single array device, such as + multi-device Btrfs or ZFS pools or `mdadm` arrays. + +This restriction does not rule out retaining storage as a named pool. +A compatible multi-device ZFS pool can be +[imported into Unraid](https://docs.unraid.net/unraid-os/advanced-configurations/optimize-storage/zfs-storage/#importing-zfs-pools-created-on-other-systems) +intact, without copying its data. Stop mergerfs and cleanly export the +pool on the source system. With the Unraid array stopped, use **Add +Pool**, assign all of the pool's devices (including support vdevs), +set **File System** to **Auto**, and start the array to import it. +If it is not recognized, investigate rather than format its devices. +The imported pool remains separate from the parity-protected array; +its redundancy comes from its ZFS layout, not Unraid array parity. + +A branch pointing at a subdirectory does **not** by itself require a +data copy: eligibility depends on its underlying disk and filesystem. +For example, importing compatible disks that contributed +`/mnt/disk1/media` and `/mnt/disk2/media` preserves their contents, and +Unraid combines the top-level `media` directories into a +[User Share](https://docs.unraid.net/unraid-os/using-unraid-to/manage-storage/shares/#user-shares). +Review the desired share layout; deeper branch directories may need +to be reorganized within their existing filesystems rather than +copied to new storage. + +### Summary + +mergerfs treats storage membership as a property of its +**configuration**, while a parity-protected Unraid array treats it as +part of the **array state**. Moving an existing filesystem into or out +of mergerfs normally does not alter that filesystem at all. Moving +populated filesystems into or out of an active Unraid parity array can +require clearing disks, rebuilding parity, moving data, or +reconstructing the array configuration. + +For semi-static data, mergerfs + SnapRAID provides pooling and parity +while retaining the ability to add and remove independently formatted +filesystems without modifying them. NonRAID provides an alternative +real-time parity layer which can likewise be combined with mergerfs. diff --git a/mkdocs/docs/project_comparisons.md b/mkdocs/docs/project_comparisons.md index 4be8d4aa..c85b831f 100644 --- a/mkdocs/docs/project_comparisons.md +++ b/mkdocs/docs/project_comparisons.md @@ -293,10 +293,16 @@ provides a similar, but not real-time, solution. fork of Unraid's parity calculation solution and can also be used with mergerfs. +Moving between the two is not symmetric because mergerfs membership is +a configuration change while Unraid array membership is part of the +array state. See [Migration](migration.md) for details. + * **Better than mergerfs for:** turnkey pooling plus integrated real-time parity in one product. -* **Worse than mergerfs for:** users who prefer a Linux distro-agnostic, - fully open-source component approach. +* **Worse than mergerfs for:** non-destructive incremental addition and + removal of populated filesystems, arbitrary branch types and filesystem + layouts, and users who prefer a Linux distro-agnostic, fully open-source + component approach. ## NonRAID diff --git a/mkdocs/mkdocs.yml b/mkdocs/mkdocs.yml index 82242318..cd9cc6d9 100644 --- a/mkdocs/mkdocs.yml +++ b/mkdocs/mkdocs.yml @@ -103,6 +103,7 @@ nav: - Tips and Notes: tips_notes.md - Known Issues and Bugs: known_issues_bugs.md - Project Comparisons: project_comparisons.md +- Migration: migration.md - Benchmarking: benchmarking.md - Performance: performance.md - Tooling: tooling.md