From baa833f13ce2a35817b35c3bcdacbe6f32cfb922 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 29 Sep 2026 09:33:22 +0200 Subject: [PATCH 01/51] Update CHECKSUMS --- CHECKSUMS | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHECKSUMS b/CHECKSUMS index b13f1562..fad5b785 100644 --- a/CHECKSUMS +++ b/CHECKSUMS @@ -215,6 +215,18 @@ e11c452e2adec376e6fcaee9ff2ec4de4e943c35690229c9c67056c5eaed52a1 snapraid-14.9- 84af790cd25c8f2d773386bca7320cb5cf81706f850eb1fda33d4b9ff4e0d692 snapraid-14.9-windows-x64.zip 7ec39de0cf7cabd09061bfc54848084b4505e77b27556850fc53cb5f23080003 snapraid-14.9-windows-x86.zip 40cee688cb1a322810eaac4e3655313c0075b791e18783bec2588084a9e47f09 snapraid-14.9-x86_64-1_am.txz +27be09151c8e779019da7d9e0501b03f56ae83da489cdaf5cd85463075cec9a2 snapraid-14.10.tar.gz +6f5a31ee04649c9b2b7b0612619f401f160c4f1a4a56947ff50381cc6036b8cc snapraid-14.10-1-x86_64.pkg.tar.zst +461778095cd40e9ef1231999dd4924499c1f6cab23e233ab811f37e29b81aefe snapraid-14.10-1.x86_64.rpm +6e8892175bfe240ae7828d513342180a249511a2d5baf8c003ae8b2bb82e0f46 snapraid-14.10-windows-x64.zip +1ec0572359fb31f2827702d792db0bdf6a95161eb9e2a6deff2bdc7e1f5b794b snapraid-14.10-windows-x86.zip +6930d629c73b5307ec32277310ed13887b1bd051e30dc41c75b4752040032d1c snapraid-14.10-x86_64-1_am.txz +1e15cc11b26b02fbb54a679c0379161bf3c94aadfaf29081d876990d809a43f9 snapraid-15.0rc2.tar.gz +a546944800a5001c3960715b97ba73cae219c854da1bb55dca16ce70fe06c6d6 snapraid-15.0rc2-1-x86_64.pkg.tar.zst +1327c122a66f91e7780e77431a318907661e690a4aca7fa78f9d504e8dde9e06 snapraid-15.0rc2-1.x86_64.rpm +092d1e91915e7644288805d1a91bd6f20ee0591d70157f5bb5980e26eaf82c1a snapraid-15.0rc2-windows-x64.zip +2c010c4ea69f39d28f266d978d78e0b2c3a845ab47f847cb5a305fc242994691 snapraid-15.0rc2-windows-x86.zip +7ada0dd95bad90895c3faaf627f99da02a4e31ac948d05f012efd2ab05a37d97 snapraid-15.0rc2-x86_64-1_am.txz 128b5b0f80a6a0911e77ed3b5d9d81529f5ea1d8fffbb5112e3af3467d80464a snapraid_14.0-1_amd64.deb b9c41ae2e8edbc413abd3172b7ed2da2cba7fa0e631d0f47fc1dafccf7c6e92c snapraid_14.1-1_amd64.deb e6456b926bb053d34645e31e5e3d8363e952c9869e873eb348abb1149598d331 snapraid_14.2-1_amd64.deb @@ -225,3 +237,5 @@ b71b556da4be6263a904fb8ac489933875467adbb920dd3911e54d87e9637ca3 snapraid_14.6- 35b156fc4d5c34339eed7e7d73e22a76e646879ecf6846b5ca20f7992ff55640 snapraid_14.7-1_amd64.deb c0e00a25b9bce40ff74f5a55401c3bfedac1881fd99aaa90a692f5858d417f7d snapraid_14.8-1_amd64.deb 09064689a64d965ac33ee89fb8b2cc14dfec0745f2b3b86a78f377fc353b7e07 snapraid_14.9-1_amd64.deb +15f5fd9f639912c94daae9d75d3567b7a3fc0e18475880f972acd8d8e9e0992c snapraid_14.10-1_amd64.deb +fc875789267ec9b057d14780f04362cc32b3cb4191e707d6e0390f05ff346d46 snapraid_15.0rc2-1_amd64.deb From 66ae70481c6b138ca3255f51f3156d8c2564a7ba Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 29 Sep 2026 10:59:57 +0200 Subject: [PATCH 02/51] Update TODO --- TODO | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/TODO b/TODO index 2c1b77a4..4623ed47 100644 --- a/TODO +++ b/TODO @@ -40,6 +40,13 @@ Future - Major +* Optimize `sync` on Btrfs by using `btrfs send -p --no-data` + to identify modified files and extent offsets, avoiding a full filesystem + metadata scan. Parity recomputation still reads from the latest snapshot. + Store the Btrfs snapshot generation (and UUID) in the content file to verify + that the parent snapshot strictly matches the content state before diffing, + falling back to a full scan if mismatched. + * Size-Based Checkpointing (autosave) using a write-ahead log (WAL). Instead of writing the full content file periodically (autosave), write incremental updates to companion `.wal` files. Use the existing bit-efficient `STREAM` From eb75cef0feb61e06cd5c6bb5d4084841d06b529a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 30 Sep 2026 15:37:52 +0200 Subject: [PATCH 03/51] Avoid touch failing on missing files deleted or moved since the last sync --- cmdline/touch.c | 13 +++++++++++-- os/mingw.c | 4 ++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmdline/touch.c b/cmdline/touch.c index 192dcdd4..dff3d3c0 100644 --- a/cmdline/touch.c +++ b/cmdline/touch.c @@ -90,6 +90,14 @@ int state_touch(struct snapraid_state* state) } #endif if (f == -1) { + /* + * Ignore missing files because they may have been deleted + * or moved since the last sync. Also ignore symbolic links + * which fail with ELOOP due to O_NOFOLLOW. + */ + if (errno == ENOENT || errno == ENOTDIR || errno == ELOOP) + continue; + /* LCOV_EXCL_START */ log_error(errno, "Error opening file '%s'. %s.\n", path, strerror(errno)); ++error; @@ -112,11 +120,12 @@ int state_touch(struct snapraid_state* state) } /* - * Touch only files that are still unchanged compared to + * Touch only regular files that are still unchanged compared to * the content file. Otherwise changing the timestamp here * could hide a modification from the following scan. */ - if (st.st_size != file->size + if (!S_ISREG(st.st_mode) + || st.st_size != file->size || st.st_mtime != file->mtime_sec || STAT_NSEC(&st) != file->mtime_nsec ) { diff --git a/os/mingw.c b/os/mingw.c index e41d15f5..50004651 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -877,6 +877,9 @@ void windows_errno(DWORD error) case ERROR_PATH_NOT_FOUND : /* in GetFileAttributeW() if internal path not found */ errno = ENOENT; break; + case ERROR_DIRECTORY : + errno = ENOTDIR; + break; case ERROR_ACCESS_DENIED : /* in CreateDirectoryW() if dir is scheduled for deletion */ case ERROR_CURRENT_DIRECTORY : /* in RemoveDirectoryW() if removing the current directory */ case ERROR_SHARING_VIOLATION : /* in RemoveDirectoryW() if in use */ @@ -4688,3 +4691,4 @@ void os_done(void) } #endif + From 426f6dc0bcf462b165ff80bcaf966e6a3082433d Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 30 Sep 2026 16:03:07 +0200 Subject: [PATCH 04/51] Treat ENOTDIR as missing file in filter_existence and handle_open_create --- cmdline/elem.c | 2 +- cmdline/handle.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 60c63a9e..2356aad6 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -375,7 +375,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub) if (lstat(path, &st) != 0) { /* if the file doesn't exist, we don't filter it out */ - if (errno == ENOENT) + if (errno == ENOENT || errno == ENOTDIR) return 0; /* LCOV_EXCL_START */ log_fatal(errno, "Error in stat file '%s'. %s.\n", path, strerror(errno)); diff --git a/cmdline/handle.c b/cmdline/handle.c index 11522209..34660054 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -113,7 +113,7 @@ int handle_open_create(struct snapraid_handle* handle, struct snapraid_file* fil * If creation was not requested, a missing file is an expected condition, * not a fatal error. */ - if (create || errno != ENOENT) { + if (create || (errno != ENOENT && errno != ENOTDIR)) { /* LCOV_EXCL_START */ log_fatal(errno, "Error opening file '%s'. %s.\n", handle->path, strerror(errno)); /* LCOV_EXCL_STOP */ From 31570ae473c51bcea66b2f42a51cc8a5a49fb755 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 30 Sep 2026 18:04:37 +0200 Subject: [PATCH 05/51] Document MuseAir and SpookyHash collision analysis --- smhasher3/README | 80 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/smhasher3/README b/smhasher3/README index 91cb73c7..74f7b5e9 100644 --- a/smhasher3/README +++ b/smhasher3/README @@ -29,13 +29,51 @@ museair_snapraid.cpp MuseAir-128.bfast-snapraid.txt SMHasher3 output for museair_snapraid.cpp. - Result comparison: + Result comparison: The SnapRAID variant passes all 252 tests. The official MuseAir BFast-128 also passes its complete test set. The different short-input path used by SnapRAID shows no statistical regression. Small differences in the p-value distribution are not significant enough to rank either version as better. + Known collision analysis: + MuseAir v2 has two known adversarial collision constructions: + + 1. Short path: A published 32-byte message pair collides with rate + ~2^-17.5 over random seeds (and a similar rate for 128-bit functions + with independent 64-bit seed halves). + + 2. Long path without compression: An algebraic product-exchange + construction produces collisions for 80, 88, and 96-byte inputs when + no 96-byte compression round is executed. For MuseAir-128, the + published 80-byte pair collides only when seed_b == 0 (a 2^-64 weak-key + condition over uniformly random seeds). + + The product-exchange construction relies on relations between the initial + state words and the finalizer products. A single 96-byte compression round + destroys these relations; the construction does not survive when preceded + by a common 96-byte block or placed within a 192-byte message. + + Relevance to SnapRAID: + The known constructions do not currently affect SnapRAID's intended use: + + - The Short path is never used: SnapRAID routes all inputs through the + Long path. + - The no-compression construction does not survive SnapRAID's block format. + SnapRAID hashes full canonical RAID blocks (with zero-padding for partial + blocks). A typical block (e.g., 256 KiB) executes thousands of 96-byte + compression rounds, which destroy the product-exchange relations. + + While SnapRAID's fixed zero seed happens to satisfy the weak-key condition + seed_b == 0, finding collisions with a known seed in non-cryptographic + hashes is expected and does not indicate a weakness for detecting random + data corruption. + + References: + https://thomasahle.com/blog/adversarial-examples-for-hashes/ + https://thomasahle.com/blog/adversarial-examples-for-hashes/records/witness-searches/reanalysis-2026-09/README.html + https://thomasahle.com/blog/adversarial-examples-for-hashes/records/witness-searches/reanalysis-2026-09/museair-v2/README.html + spooky2_snapraid.cpp SpookyHash V2 128-bit as used by SnapRAID. SnapRAID always uses the Long path instead of the standard Short path below 192 bytes. @@ -43,12 +81,48 @@ spooky2_snapraid.cpp SpookyHash2-128-snapraid.txt SMHasher3 output for spooky2_snapraid.cpp. - Result comparison: + Result comparison: The SnapRAID variant passes all 252 tests. The official SpookyHash V2 128-bit implementation has strong BIC failures on several short input lengths. Those failures come from the Short path. Always using the Long path, as SnapRAID does, removes the observed BIC weakness. + Known collision analysis: + An adversarial collision has been demonstrated in the SpookyHash V2 Long + path itself. Published 275- and 286-byte message pairs introduce controlled + differences in the final 96-byte Mix block and cancel them in the End block. + + The required sign of the correction depends on a single internal state bit. + Two complementary pairs cover both possible values of this bit, each + colliding for approximately half (~2^-1) of randomly tested seeds. When the + cancellation succeeds, the entire 768-bit internal state matches before the + finalization rounds, yielding an identical 128-bit hash. + + Because the differential is injected in the final Mix block, the attack can + be extended to larger inputs by prepending common 96-byte blocks, provided + the final block preserves the required offset. + + Relevance to SnapRAID: + SnapRAID stores its 128-bit SpookyHash seed in the content file. Because the + seed is public, an attacker can determine the appropriate variant rather + than relying on a random ~2^-1 success rate. + + Furthermore, unlike MuseAir, SpookyHash does not pad partial blocks. A final + file block can therefore have arbitrary lengths (such as 275 or 286 bytes), + making the published lengths directly applicable. For full blocks, extending + the attack requires common 96-byte prefixes that preserve the required End + block structure. + + However, finding collisions with a public seed in non-cryptographic hashes + is expected and does not impact SnapRAID's intended use: accidental storage + corruption remains random and is not crafted to exploit the known seed or + internal hash state. + + References: + https://thomasahle.com/blog/adversarial-examples-for-hashes/ + https://thomasahle.com/blog/adversarial-examples-for-hashes/verify/spookyhash2-64/README.html + https://github.com/thomasahle/hash-collision-witnesses/tree/main/spookyhash2-64 + murmur3_snapraid.cpp MurmurHash3 x86_128 as used by SnapRAID, with the SnapRAID 128-bit seed interface. @@ -56,7 +130,7 @@ murmur3_snapraid.cpp MurmurHash3-128-snapraid.txt SMHasher3 output for murmur3_snapraid.cpp. - Result comparison: + Result comparison: The SnapRAID variant passes 179 of 252 tests, compared with 154 of 250 for the official MurmurHash3 x86_128 int32 variant. Most data-path failures are the same, including BIC, Text, TwoBytes, PerlinNoise and From edb3cacb77c225eba102e19b0af7494747d1e711 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 30 Sep 2026 13:31:34 +0200 Subject: [PATCH 06/51] Optimize hash performance with custom unroll --- Makefile.am | 5 ++++- cmdline/murmur3.c | 39 +++++++++------------------------------ cmdline/museair.c | 5 +++++ cmdline/spooky2.c | 7 ++++++- cmdline/util.h | 5 +++++ 5 files changed, 29 insertions(+), 32 deletions(-) diff --git a/Makefile.am b/Makefile.am index 471f18b5..5d2a06a2 100644 --- a/Makefile.am +++ b/Makefile.am @@ -217,7 +217,10 @@ $(CLI_PREFIX)sync.$(OBJEXT): $(CLI_INCLUDES) $(CLI_PREFIX)thermal.$(OBJEXT): $(CLI_INCLUDES) $(CLI_PREFIX)touch.$(OBJEXT): $(CLI_INCLUDES) $(CLI_PREFIX)unixapp.$(OBJEXT): $(CLI_INCLUDES) -$(CLI_PREFIX)util.$(OBJEXT): $(CLI_INCLUDES) +$(CLI_PREFIX)util.$(OBJEXT): $(CLI_INCLUDES) \ + $(srcdir)/cmdline/murmur3.c \ + $(srcdir)/cmdline/spooky2.c \ + $(srcdir)/cmdline/museair.c # Add the .version file in the distribution dist-hook: diff --git a/cmdline/murmur3.c b/cmdline/murmur3.c index 9ac614db..51cc8b09 100644 --- a/cmdline/murmur3.c +++ b/cmdline/murmur3.c @@ -26,42 +26,16 @@ static inline uint32_t fmix32(uint32_t h) } /* - * Warning! - * Don't declare these variables static, otherwise the gcc optimizer - * may generate very slow code for multiplication with these constants, - * like: - - -> .cpp - k1 *= c1; - -> .asm - 152: 8d 14 80 lea (%eax,%eax,4),%edx - 155: 8d 14 90 lea (%eax,%edx,4),%edx - 158: c1 e2 03 shl $0x3,%edx - 15b: 29 c2 sub %eax,%edx - 15d: 8d 14 d2 lea (%edx,%edx,8),%edx - 160: 8d 14 90 lea (%eax,%edx,4),%edx - 163: 8d 14 d0 lea (%eax,%edx,8),%edx - 166: 8d 14 90 lea (%eax,%edx,4),%edx - 169: 8d 14 50 lea (%eax,%edx,2),%edx - 16c: 8d 14 90 lea (%eax,%edx,4),%edx - 16f: 8d 14 92 lea (%edx,%edx,4),%edx - 172: 8d 14 50 lea (%eax,%edx,2),%edx - 175: 8d 04 d0 lea (%eax,%edx,8),%eax - 178: 8d 14 c5 00 00 00 00 lea 0x0(,%eax,8),%edx - 17f: 29 d0 sub %edx,%eax - - * resulting in speeds of 500 MB/s instead of 3000 MB/s. - * - * Verified with gcc 4.4.4 compiling with : - * - * g++ -g -c -O2 MurmurHash3.cpp -o MurmurHash3.o + * Non-static variables are intentionally used instead of static const + * because keeping these constants in CPU registers yields measurably + * higher throughput than using 32-bit immediate constants in imul. */ uint32_t c1 = 0x239b961b; uint32_t c2 = 0xab0e9789; uint32_t c3 = 0x38b34ae5; uint32_t c4 = 0xa1e38b93; -void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, void* digest) +static void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, void* digest) { const uint8_t* p; const uint8_t* end; @@ -77,6 +51,11 @@ void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, voi end = p + (size & ~15); /* body */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC unroll 8 +#elif defined(__clang__) +#pragma unroll 8 +#endif while (p < end) { uint32_t k1 = util_read32(p + 0); uint32_t k2 = util_read32(p + 4); diff --git a/cmdline/museair.c b/cmdline/museair.c index e6db6046..1eaad941 100644 --- a/cmdline/museair.c +++ b/cmdline/museair.c @@ -190,6 +190,11 @@ static void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uin state[5] ^= seed_b & MASK_K; if (unlikely(q > u64x(12))) { +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC unroll 2 +#elif defined(__clang__) +#pragma unroll 2 +#endif do { state[0] ^= util_read64(p + u64x(0)); state[1] ^= util_read64(p + u64x(1)); diff --git a/cmdline/spooky2.c b/cmdline/spooky2.c index ca063a32..67a86efe 100644 --- a/cmdline/spooky2.c +++ b/cmdline/spooky2.c @@ -96,7 +96,7 @@ // #define sc_const 0xdeadbeefdeadbeefULL -void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) +static void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) { uint64_t h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11; uint8_t buf[sc_blockSize]; @@ -117,6 +117,11 @@ void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* end = p + nblocks * sc_blockSize; /* body */ +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC unroll 1 +#elif defined(__clang__) +#pragma unroll 1 +#endif while (p < end) { Mix(p, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); p += sc_blockSize; diff --git a/cmdline/util.h b/cmdline/util.h index f65dce31..a197f01f 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -110,6 +110,11 @@ uint32_t crc32c_shift(uint32_t crc, size_t size); */ static inline uint32_t crc32c_gen_plain(uint32_t crc, const unsigned char* ptr, size_t size) { +#if defined(__GNUC__) && !defined(__clang__) +#pragma GCC unroll 1 +#elif defined(__clang__) +#pragma unroll 1 +#endif while (size >= 4) { crc ^= ptr[0] | (uint32_t)ptr[1] << 8 | (uint32_t)ptr[2] << 16 | (uint32_t)ptr[3] << 24; crc = CRC32C_3[crc & 0xff] ^ CRC32C_2[(crc >> 8) & 0xff] ^ CRC32C_1[(crc >> 16) & 0xff] ^ CRC32C_0[crc >> 24]; From 5b402fc46232f0cd7162c348e71d8f2656b45f7a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2026 08:41:43 +0200 Subject: [PATCH 07/51] Normalize legacy out-of-range content nanoseconds to STAT_NSEC_INVALID --- cmdline/state.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/cmdline/state.c b/cmdline/state.c index c3798beb..29d4d4ab 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2548,15 +2548,20 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } ret = sgetb32(f, &v_mtime_nsec); - if (ret < 0 || v_mtime_nsec > 1000000000) { + if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); os_abort(); /* LCOV_EXCL_STOP */ } - /* STAT_NSEC_INVALID is encoded as 0 */ - if (v_mtime_nsec == 0) + /* + * STAT_NSEC_INVALID is encoded as 0. + * Values > 1000000000 can occur in content files written by SnapRAID <= 14.10 + * on Windows for files with pre-1970 timestamps due to negative nanoseconds. + * Normalize them to STAT_NSEC_INVALID. + */ + if (v_mtime_nsec == 0 || v_mtime_nsec > 1000000000) v_mtime_nsec = STAT_NSEC_INVALID; else --v_mtime_nsec; @@ -3008,15 +3013,20 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } ret = sgetb32(f, &v_mtime_nsec); - if (ret < 0 || v_mtime_nsec > 1000000000) { + if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); os_abort(); /* LCOV_EXCL_STOP */ } - /* STAT_NSEC_INVALID is encoded as 0 */ - if (v_mtime_nsec == 0) + /* + * STAT_NSEC_INVALID is encoded as 0. + * Values > 1000000000 can occur in content files written by SnapRAID <= 14.10 + * on Windows for files with pre-1970 timestamps due to negative nanoseconds. + * Normalize them to STAT_NSEC_INVALID. + */ + if (v_mtime_nsec == 0 || v_mtime_nsec > 1000000000) v_mtime_nsec = STAT_NSEC_INVALID; else --v_mtime_nsec; From 70947d4054608d53cd0f862c5007428321b9b134 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2026 08:55:44 +0200 Subject: [PATCH 08/51] Print abort stacktrace to stderr and exit with code 134 on Windows --- os/mingw.c | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/os/mingw.c b/os/mingw.c index 50004651..3341e784 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -4537,26 +4537,27 @@ void os_abort(void) size_t size; unsigned i; - printf("Stacktrace of " PACKAGE " v" VERSION); - printf(", mingw"); + fprintf(stderr, "Stacktrace of " PACKAGE " v" VERSION); + fprintf(stderr, ", mingw"); #ifdef __GNUC__ - printf(", gcc " __VERSION__); + fprintf(stderr, ", gcc " __VERSION__); #endif - printf(", %d-bit", (int)sizeof(void*) * 8); - printf(", PATH_MAX=%d", PATH_MAX); - printf("\n"); + fprintf(stderr, ", %d-bit", (int)sizeof(void*) * 8); + fprintf(stderr, ", PATH_MAX=%d", PATH_MAX); + fprintf(stderr, "\n"); /* get stackstrace, but without symbols */ - size = CaptureStackBackTrace(0, 32, stack, NULL); + size = CaptureStackBackTrace(0, 32, stack, 0); for (i = 0; i < size; ++i) - printf("[bt] %02u: %p\n", i, stack[i]); + fprintf(stderr, "[bt] %02u: %p\n", i, stack[i]); - printf("Please report this error to the SnapRAID Issues:\n"); - printf("https://github.com/amadvance/snapraid/issues\n"); + fprintf(stderr, "Please report this error to the SnapRAID Issues:\n"); + fprintf(stderr, "https://github.com/amadvance/snapraid/issues\n"); + fflush(stderr); - /* use exit() and not abort to avoid the Windows abort dialog */ - os_exit(); + /* use exit(134) (128 + SIGABRT) and not abort to avoid the Windows abort dialog while distinguishing abort from normal failure */ + exit(134); } void os_exit(void) From 0b930b8a31de3b71bd37bdef3a082d32dd4b7f4c Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2026 09:05:01 +0200 Subject: [PATCH 09/51] Add unique descriptive log_fatal messages with field names to all decoding_error calls --- cmdline/elem.c | 4 +- cmdline/parity.c | 4 +- cmdline/state.c | 180 +++++++++++++++++++++++++++++------------------ cmdline/util.c | 2 +- 4 files changed, 116 insertions(+), 74 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 2356aad6..24662185 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -904,7 +904,7 @@ void dealloc_import(tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struc if (parity_pos == POS_NULL) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: Missing parity position for dealloc block %" PRIu64 "\n", i); - exit(EXIT_FAILURE); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -920,7 +920,7 @@ void dealloc_import(tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struc default : /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: State for dealloc block %" PRIu64 " state %u\n", i, block_state); - exit(EXIT_FAILURE); + os_abort(); /* LCOV_EXCL_STOP */ } } diff --git a/cmdline/parity.c b/cmdline/parity.c index 5ea3b735..c572eb36 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -625,7 +625,7 @@ int parity_restore(struct snapraid_parity_handle* handle, data_off_t size, uint3 if ((run & block_mask) != 0) { /* LCOV_EXCL_START */ errno = ENXIO; - log_fatal(EINTERNAL, "Internal inconsistency in split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); + log_fatal(EINTERNAL, "Split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); return -1; /* LCOV_EXCL_STOP */ } @@ -731,7 +731,7 @@ int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* if ((run & block_mask) != 0) { /* LCOV_EXCL_START */ errno = ENXIO; - log_fatal(EINTERNAL, "Internal inconsistency in split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); + log_fatal(EINTERNAL, "Split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); return -1; /* LCOV_EXCL_STOP */ } diff --git a/cmdline/state.c b/cmdline/state.c index 29d4d4ab..781a77c4 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2428,7 +2428,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid header!\n"); + log_fatal(ECONTENT, "Invalid header: read failed!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2457,7 +2457,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_START */ if (memcmp(buffer, "SNAPCNT", 7) != 0) { decoding_error(path, f); - log_fatal(ECONTENT, "Invalid header!\n"); + log_fatal(ECONTENT, "Invalid header: magic mismatch!\n"); os_abort(); } else { log_fatal(ECONTENT, "The content file '%s' was generated with a newer version of SnapRAID!\n", path); @@ -2491,7 +2491,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: File mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2501,6 +2501,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2509,23 +2510,23 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid file size %" PRIu64 "!\n", v_size); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'f' size %" PRIu64 " exceeds maximum!\n", v_size); + os_abort(); /* LCOV_EXCL_STOP */ } if (state->block_size == 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Zero blocksize\n"); - exit(EXIT_FAILURE); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' encountered with zero blocksize\n"); + os_abort(); /* LCOV_EXCL_STOP */ } if (!has_blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Missing 'blockmax' before file in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' missing 'blockmax' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2534,7 +2535,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size / state->block_size > blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: File size too big!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' size is too big!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2543,6 +2544,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid mtime_sec\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2551,6 +2553,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid mtime_nsec\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2570,6 +2573,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid inode\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2578,13 +2582,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Invalid file '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has invalid subpath '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2618,6 +2623,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block run has an invalid pos\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2626,6 +2632,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block run has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2633,7 +2640,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count == 0 || v_count > file->blockmax || v_idx > file->blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Block number out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block number out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2641,7 +2648,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count > blockmax || v_pos > blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Block size %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block position out of range %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2669,7 +2676,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid block type!\n"); + log_fatal(ECONTENT, "Internal inconsistency: Field 'f' has invalid block type '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2683,6 +2690,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' error reading block run\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2703,7 +2711,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_info || !has_blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'info' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' duplicate or misplaced in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2713,6 +2721,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' has an invalid oldest timestamp\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2721,8 +2730,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_oldest > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid info oldest timestamp %" PRIu64 "!\n", v_oldest); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'i' has an out-of-range oldest timestamp %" PRIu64 "!\n", v_oldest); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -2739,6 +2748,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' run has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2746,7 +2756,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count > blockmax || v_pos > blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Info size %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' run size out of range %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2755,6 +2765,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' has invalid flags\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2766,6 +2777,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' has an invalid timestamp\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2774,8 +2786,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (t64 > INT64_MAX - v_oldest) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid info timestamp %" PRIu64 "!\n", t64); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'i' has an overflowing timestamp %" PRIu64 "!\n", t64); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -2794,7 +2806,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (rehash && state->prevhash == HASH_UNDEFINED) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Missing previous checksum!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' requires previous checksum but none defined\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2803,7 +2815,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!info) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Missing info!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' failed to create info object\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2819,7 +2831,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (fs_info_is_required(state, v_pos)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Missing info!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' missing required info for position %" PRIu64 "\n", (uint64_t)v_pos); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2850,7 +2862,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Hole mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2859,7 +2871,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!has_blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Missing 'blockmax' before hole in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' missing 'blockmax' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2873,6 +2885,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' run has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2880,7 +2893,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count == 0 || v_count > blockmax || v_pos > blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Hole size %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' size out of range %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2897,8 +2910,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count > INT64_MAX / (uint64_t)state->block_size) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid deleted block count %" PRIu64 "!\n", v_count); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'h' deleted block count %" PRIu64 " overflows size!\n", v_count); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -2919,6 +2932,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' error reading deleted block run\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2934,7 +2948,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid hole type!\n"); + log_fatal(ECONTENT, "Internal inconsistency: Field 'h' has invalid hole type '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2950,7 +2964,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Dealloc mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2960,6 +2974,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2976,13 +2991,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Invalid dealloc '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has invalid subpath '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2991,6 +3007,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2999,8 +3016,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid deallocated file size %" PRIu64 "!\n", v_size); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'd' entry has an out-of-range size %" PRIu64 "!\n", v_size); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -3008,6 +3025,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid mtime_sec\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3016,6 +3034,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid mtime_nsec\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3047,7 +3066,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (state->prevhash == HASH_UNDEFINED) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Previous dealloc hash without previous checksum!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' previous hash without previous checksum\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3055,7 +3074,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } else { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid dealloc hash type!\n"); + log_fatal(ECONTENT, "Internal inconsistency: Field 'd' has invalid hash type '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3064,6 +3083,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' error reading block hash\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3084,7 +3104,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Symlink mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 's' mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3094,6 +3114,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 's' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3101,7 +3122,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Invalid symlink '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Field 's' has invalid subpath '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3110,6 +3131,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 's' has an invalid target path\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3135,7 +3157,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Hardlink mapping index out of range!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3145,6 +3167,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3152,7 +3175,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Invalid hardlink '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has invalid subpath '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3161,6 +3184,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has an invalid target path\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3168,7 +3192,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!path_is_sub(linkto)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Invalid hardlink target '%s' for '%s'!\n", linkto, sub); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has invalid target '%s' for '%s'!\n", linkto, sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3193,7 +3217,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Dir mapping index out of range!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'r' mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3203,6 +3227,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'r' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3210,7 +3235,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Invalid dir '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'r' has invalid subpath '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3228,7 +3253,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_hash || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'hash' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'c' duplicate or misplaced the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3247,7 +3272,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid checksum!\n"); + log_fatal(ECONTENT, "Internal inconsistency: Field 'c' has invalid checksum type '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3257,6 +3282,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'c' error reading hash seed\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3264,7 +3290,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_prevhash || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'prevhash' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'C' duplicate or misplaced in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3283,7 +3309,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid checksum!\n"); + log_fatal(ECONTENT, "Internal inconsistency: Field 'C' has invalid checksum type '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3293,6 +3319,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'C' error reading hash seed\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3302,7 +3329,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_block_size || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'blocksize' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'z' duplicate or misplaced in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3312,6 +3339,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'z' has an invalid block size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3319,8 +3347,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (block_size == 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Zero 'blocksize' specification in the content file!\n"); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'z' has zero 'blocksize' specification in the content file!\n"); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -3343,7 +3371,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_hash_size || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'hashsize' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'y' duplicate or misplaced in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3353,6 +3381,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'y' has an invalid hash size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3368,7 +3397,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (hash_size < 4 || hash_size > HASH_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid 'hashsize' specification in the content file!\n"); + log_fatal(ECONTENT, "Invalid 'hashsize' specification %" PRIu64 " in the content file!\n", hash_size); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3390,7 +3419,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_blockmax || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'blockmax' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'x' duplicate or misplaced in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3400,6 +3429,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'x' has an invalid blockmax\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3434,6 +3464,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid disk name\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3442,6 +3473,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid disk index\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3449,8 +3481,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_idx >= RAID_DATA_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid disk position '%u' in the content file!\n", v_idx); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'M' invalid disk position '%u' in the content file!\n", v_idx); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -3458,6 +3490,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid total_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3466,6 +3499,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid free_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3475,6 +3509,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid uuid\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3523,6 +3558,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid parity level\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3531,6 +3567,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid total_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3539,6 +3576,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid free_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3547,6 +3585,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid split count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3554,16 +3593,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_split_mac > SPLIT_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Too many splits '%u' in the content file!\n", v_split_mac); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' has too many splits '%u' in the content file!\n", v_split_mac); + os_abort(); /* LCOV_EXCL_STOP */ } if (v_level >= LEV_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid parity level '%u' in the configuration file!\n", v_level); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' has invalid parity level '%u' in the content file!\n", v_level); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -3591,6 +3630,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' split has an invalid path\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3599,6 +3639,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' split has an invalid uuid\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3607,6 +3648,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); + log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' split has an invalid size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3615,16 +3657,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid split file size %" PRIu64 "!\n", v_size); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' split has an out-of-range size %" PRIu64 "!\n", v_size); + os_abort(); /* LCOV_EXCL_STOP */ } if (v_size > INT64_MAX - v_size_total) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Total parity split size is too large!\n"); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' total parity split size is too large!\n"); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -3633,16 +3675,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!has_block_size) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Missing 'blocksize' before parity split information in the content file!\n"); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' missing 'blocksize' parity split information in the content file!\n"); + os_abort(); /* LCOV_EXCL_STOP */ } if (v_size % state->block_size != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid parity split size %" PRIu64 " not aligned to block size %u!\n", v_size, state->block_size); - exit(EXIT_FAILURE); + log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' parity split size %" PRIu64 " not aligned to block size %u!\n", v_size, state->block_size); + os_abort(); /* LCOV_EXCL_STOP */ } @@ -3723,7 +3765,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } else { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid command '%c'!\n", (char)c); + log_fatal(ECONTENT, "Internal inconsistency: Invalid field command '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3782,7 +3824,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* check that the stored parity size matches the loaded state */ if (blockmax != parity_allocated_size(state)) { /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "Internal inconsistency: Parity size %" PRIu64 "/%" PRIu64 " in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f)); + log_fatal(EINTERNAL, "Wrong parity size %" PRIu64 "/%" PRIu64 " in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f)); if (state->opt.skip_content_check) { log_fatal(ECONTENT, "Overriding.\n"); blockmax = parity_allocated_size(state); @@ -4004,7 +4046,7 @@ static void* state_write_thread(void* arg) if (!disk) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: Unmapped disk '%s'\n", map->name); - goto bail; + os_abort(); /* LCOV_EXCL_STOP */ } @@ -4165,7 +4207,7 @@ static void* state_write_thread(void* arg) default : /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: State for block %" PRIu64 " state %u\n", v_pos, v_state); - goto bail; + os_abort(); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/util.c b/cmdline/util.c index 233656f8..e1f7ffe3 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -826,7 +826,7 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* default : /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency in hash function %u\n", kind); - exit(EXIT_FAILURE); + os_abort(); break; /* LCOV_EXCL_STOP */ } From e42876e6205ee34c8f3ec0da638990ffb05ce3c7 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2026 19:10:19 +0200 Subject: [PATCH 10/51] Update HISTORY --- HISTORY | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY b/HISTORY index 18db39ff..29bf0a47 100644 --- a/HISTORY +++ b/HISTORY @@ -40,7 +40,7 @@ SnapRAID HISTORY behaviour is preserved. Primarily intended for NixOS and other non-FHS systems. * Removed the 512 TiB limitation on hard disk size (when using the - default 128 KiB blocks). + default 256 KiB blocks). * Added support for a custom info options tag `[info: ...]` in the `smartctl` configuration option. This allows replacing the default `-a` command line argument used to retrieve SMART attributes with a From 68b94bfdf58b80435a1da3cc78457f7ba7e82000 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2026 19:55:23 +0200 Subject: [PATCH 11/51] Document interaction between filesystem snapshots and external backup tools --- doc/snapraid-de.1 | 31 +++++++++++++++++++++++++++++++ doc/snapraid-de.d | 32 ++++++++++++++++++++++++++++++++ doc/snapraid-de.txt | 34 ++++++++++++++++++++++++++++++++++ doc/snapraid-es.1 | 30 ++++++++++++++++++++++++++++++ doc/snapraid-es.d | 31 +++++++++++++++++++++++++++++++ doc/snapraid-es.txt | 33 +++++++++++++++++++++++++++++++++ doc/snapraid-fr.1 | 30 ++++++++++++++++++++++++++++++ doc/snapraid-fr.d | 31 +++++++++++++++++++++++++++++++ doc/snapraid-fr.txt | 33 +++++++++++++++++++++++++++++++++ doc/snapraid-it.1 | 28 ++++++++++++++++++++++++++++ doc/snapraid-it.d | 29 +++++++++++++++++++++++++++++ doc/snapraid-it.txt | 31 +++++++++++++++++++++++++++++++ doc/snapraid-ja.1 | 25 +++++++++++++++++++++++++ doc/snapraid-ja.d | 26 ++++++++++++++++++++++++++ doc/snapraid-ja.txt | 28 ++++++++++++++++++++++++++++ doc/snapraid-ko.1 | 22 ++++++++++++++++++++++ doc/snapraid-ko.d | 23 +++++++++++++++++++++++ doc/snapraid-ko.txt | 25 +++++++++++++++++++++++++ doc/snapraid-pl.1 | 30 ++++++++++++++++++++++++++++++ doc/snapraid-pl.d | 31 +++++++++++++++++++++++++++++++ doc/snapraid-pl.txt | 33 +++++++++++++++++++++++++++++++++ doc/snapraid-pt.1 | 30 ++++++++++++++++++++++++++++++ doc/snapraid-pt.d | 31 +++++++++++++++++++++++++++++++ doc/snapraid-pt.txt | 33 +++++++++++++++++++++++++++++++++ doc/snapraid-ro.1 | 30 ++++++++++++++++++++++++++++++ doc/snapraid-ro.d | 31 +++++++++++++++++++++++++++++++ doc/snapraid-ro.txt | 33 +++++++++++++++++++++++++++++++++ doc/snapraid-ru.1 | 28 ++++++++++++++++++++++++++++ doc/snapraid-ru.d | 29 +++++++++++++++++++++++++++++ doc/snapraid-ru.txt | 31 +++++++++++++++++++++++++++++++ doc/snapraid-sv.1 | 28 ++++++++++++++++++++++++++++ doc/snapraid-sv.d | 29 +++++++++++++++++++++++++++++ doc/snapraid-sv.txt | 31 +++++++++++++++++++++++++++++++ doc/snapraid-uk.1 | 28 ++++++++++++++++++++++++++++ doc/snapraid-uk.d | 29 +++++++++++++++++++++++++++++ doc/snapraid-uk.txt | 31 +++++++++++++++++++++++++++++++ doc/snapraid-zh.1 | 19 +++++++++++++++++++ doc/snapraid-zh.d | 20 ++++++++++++++++++++ doc/snapraid-zh.txt | 22 ++++++++++++++++++++++ doc/snapraid.1 | 27 +++++++++++++++++++++++++++ doc/snapraid.d | 28 ++++++++++++++++++++++++++++ doc/snapraid.txt | 30 ++++++++++++++++++++++++++++++ 42 files changed, 1214 insertions(+) diff --git a/doc/snapraid-de.1 b/doc/snapraid-de.1 index f2714b9f..a64fdd51 100644 --- a/doc/snapraid-de.1 +++ b/doc/snapraid-de.1 @@ -2075,6 +2075,37 @@ Wenn ein \`sync\` jedoch unterbrochen wird, vermeiden Sie es, Dateien zu die aus dem vom unterbrochenen Sync verwendeten Snapshot berechnet wurden. Ein späterer Sync könnte diesen Snapshot ersetzen, wodurch Zwischenversionen von Dateien für die Wiederherstellung nicht mehr verfügbar sind. +.SS Externe Sicherungen mit Dateisystem\-Snapshots (External backups with filesystem snapshots) +Wenn Dateisystem\-Snapshots aktiviert sind, greift SnapRAID auf Snapshots +an der Wurzel jedes Daten\-Subvolumes, Volumes oder Datasets zu oder erstellt +diese dort. +.PP +Ein einfacher Weg, um jegliche Beeinträchtigung durch externe Tools zu +vermeiden, besteht darin, die \`data\`\-Konfiguration auf ein reguläres +Unterverzeichnis innerhalb des Subvolumes oder Volumes verweisen zu lassen +(beispielsweise \`/mnt/subvol/data\` statt \`/mnt/subvol\`). SnapRAID erstellt +das Snapshot\-Verzeichnis immer an der Wurzel des Subvolumes (beispielsweise +\`/mnt/subvol/.snapraid/\`), wodurch das Datenunterverzeichnis sauber bleibt +und Snapshots außerhalb und unsichtbar für Sicherungswerkzeuge bleiben, die +darin arbeiten. +.PP +Wenn das konfigurierte \`data\`\-Verzeichnis hingegen selbst die Wurzel des +Subvolumes oder Volumes ist, erstellt SnapRAID ein \`.snapraid/\`\-Verzeichnis +direkt darin (auf Btrfs, Bcachefs und NTFS). Während SnapRAID dieses +Verzeichnis bei seinen eigenen Scans automatisch ausschließt, werden +externe Sicherungs\- oder Synchronisationswerkzeuge darauf stoßen. Wenn es +nicht ausgeschlossen wird, könnten sie in die Snapshots oder Verknüpfungen +eindringen und den gesamten eingefrorenen Datenbaum kopieren, was Daten +dupliziert und den Sicherungsspeicherplatz erschöpfen kann. Konfigurieren +Sie externe Sicherungswerkzeuge immer so, dass sie das Verzeichnis +\`.snapraid/\` ausschließen. +.PP +Bei ZFS werden Snapshots über das standardmäßige virtuelle Verzeichnis +\`.zfs/snapshot/\` anstelle von \`.snapraid/\` bereitgestellt. Standardmäßig +(\`snapdir=hidden\`) blendet ZFS dieses Verzeichnis in Verzeichnislisten aus, +sodass externe Sicherungswerkzeuge es nicht durchlaufen. Wenn Sie +\`snapdir=visible\` aktiviert haben, stellen Sie sicher, dass Ihre +Sicherungswerkzeuge das Verzeichnis \`.zfs/\` ausschließen. .SH MUSTER (PATTERN) Muster bieten eine flexible Möglichkeit, Dateien für die Einbeziehung oder den Ausschluss zu filtern. Durch die Verwendung von Globbing\-Zeichen können Sie Regeln definieren, diff --git a/doc/snapraid-de.d b/doc/snapraid-de.d index 70c33437..3371c773 100644 --- a/doc/snapraid-de.d +++ b/doc/snapraid-de.d @@ -1669,6 +1669,38 @@ Lebenszyklus von Snapshots (Snapshots Lifecycle) Ein späterer Sync könnte diesen Snapshot ersetzen, wodurch Zwischenversionen von Dateien für die Wiederherstellung nicht mehr verfügbar sind. + Externe Sicherungen mit Dateisystem-Snapshots (External backups with filesystem snapshots) + Wenn Dateisystem-Snapshots aktiviert sind, greift SnapRAID auf Snapshots + an der Wurzel jedes Daten-Subvolumes, Volumes oder Datasets zu oder erstellt + diese dort. + + Ein einfacher Weg, um jegliche Beeinträchtigung durch externe Tools zu + vermeiden, besteht darin, die `data`-Konfiguration auf ein reguläres + Unterverzeichnis innerhalb des Subvolumes oder Volumes verweisen zu lassen + (beispielsweise `/mnt/subvol/data` statt `/mnt/subvol`). SnapRAID erstellt + das Snapshot-Verzeichnis immer an der Wurzel des Subvolumes (beispielsweise + `/mnt/subvol/.snapraid/`), wodurch das Datenunterverzeichnis sauber bleibt + und Snapshots außerhalb und unsichtbar für Sicherungswerkzeuge bleiben, die + darin arbeiten. + + Wenn das konfigurierte `data`-Verzeichnis hingegen selbst die Wurzel des + Subvolumes oder Volumes ist, erstellt SnapRAID ein `.snapraid/`-Verzeichnis + direkt darin (auf Btrfs, Bcachefs und NTFS). Während SnapRAID dieses + Verzeichnis bei seinen eigenen Scans automatisch ausschließt, werden + externe Sicherungs- oder Synchronisationswerkzeuge darauf stoßen. Wenn es + nicht ausgeschlossen wird, könnten sie in die Snapshots oder Verknüpfungen + eindringen und den gesamten eingefrorenen Datenbaum kopieren, was Daten + dupliziert und den Sicherungsspeicherplatz erschöpfen kann. Konfigurieren + Sie externe Sicherungswerkzeuge immer so, dass sie das Verzeichnis + `.snapraid/` ausschließen. + + Bei ZFS werden Snapshots über das standardmäßige virtuelle Verzeichnis + `.zfs/snapshot/` anstelle von `.snapraid/` bereitgestellt. Standardmäßig + (`snapdir=hidden`) blendet ZFS dieses Verzeichnis in Verzeichnislisten aus, + sodass externe Sicherungswerkzeuge es nicht durchlaufen. Wenn Sie + `snapdir=visible` aktiviert haben, stellen Sie sicher, dass Ihre + Sicherungswerkzeuge das Verzeichnis `.zfs/` ausschließen. + Muster (Pattern) Muster bieten eine flexible Möglichkeit, Dateien für die Einbeziehung oder den Ausschluss zu filtern. Durch die Verwendung von Globbing-Zeichen können Sie Regeln definieren, diff --git a/doc/snapraid-de.txt b/doc/snapraid-de.txt index 4793db60..75776832 100644 --- a/doc/snapraid-de.txt +++ b/doc/snapraid-de.txt @@ -1779,6 +1779,40 @@ die aus dem vom unterbrochenen Sync verwendeten Snapshot berechnet wurden. Ein späterer Sync könnte diesen Snapshot ersetzen, wodurch Zwischenversionen von Dateien für die Wiederherstellung nicht mehr verfügbar sind. +9.1 Externe Sicherungen mit Dateisystem-Snapshots (External backups with filesystem snapshots) +---------------------------------------------------------------------------------------------- + +Wenn Dateisystem-Snapshots aktiviert sind, greift SnapRAID auf Snapshots +an der Wurzel jedes Daten-Subvolumes, Volumes oder Datasets zu oder erstellt +diese dort. + +Ein einfacher Weg, um jegliche Beeinträchtigung durch externe Tools zu +vermeiden, besteht darin, die `data`-Konfiguration auf ein reguläres +Unterverzeichnis innerhalb des Subvolumes oder Volumes verweisen zu lassen +(beispielsweise `/mnt/subvol/data` statt `/mnt/subvol`). SnapRAID erstellt +das Snapshot-Verzeichnis immer an der Wurzel des Subvolumes (beispielsweise +`/mnt/subvol/.snapraid/`), wodurch das Datenunterverzeichnis sauber bleibt +und Snapshots außerhalb und unsichtbar für Sicherungswerkzeuge bleiben, die +darin arbeiten. + +Wenn das konfigurierte `data`-Verzeichnis hingegen selbst die Wurzel des +Subvolumes oder Volumes ist, erstellt SnapRAID ein `.snapraid/`-Verzeichnis +direkt darin (auf Btrfs, Bcachefs und NTFS). Während SnapRAID dieses +Verzeichnis bei seinen eigenen Scans automatisch ausschließt, werden +externe Sicherungs- oder Synchronisationswerkzeuge darauf stoßen. Wenn es +nicht ausgeschlossen wird, könnten sie in die Snapshots oder Verknüpfungen +eindringen und den gesamten eingefrorenen Datenbaum kopieren, was Daten +dupliziert und den Sicherungsspeicherplatz erschöpfen kann. Konfigurieren +Sie externe Sicherungswerkzeuge immer so, dass sie das Verzeichnis +`.snapraid/` ausschließen. + +Bei ZFS werden Snapshots über das standardmäßige virtuelle Verzeichnis +`.zfs/snapshot/` anstelle von `.snapraid/` bereitgestellt. Standardmäßig +(`snapdir=hidden`) blendet ZFS dieses Verzeichnis in Verzeichnislisten aus, +sodass externe Sicherungswerkzeuge es nicht durchlaufen. Wenn Sie +`snapdir=visible` aktiviert haben, stellen Sie sicher, dass Ihre +Sicherungswerkzeuge das Verzeichnis `.zfs/` ausschließen. + 10 MUSTER (PATTERN) =================== diff --git a/doc/snapraid-es.1 b/doc/snapraid-es.1 index ff245ca7..6d8294a0 100644 --- a/doc/snapraid-es.1 +++ b/doc/snapraid-es.1 @@ -2034,6 +2034,36 @@ con éxito. La paridad aún puede contener bloques calculados a partir de la instantánea utilizada por el sync interrumpido. Un sync posterior puede reemplazar esa instantánea, haciendo que las versiones intermedias de los archivos no estén disponibles para la recuperación. +.SS Copias de seguridad externas con instantáneas del sistema de archivos (External backups with filesystem snapshots) +Cuando las instantáneas del sistema de archivos están habilitadas, SnapRAID +accede o crea instantáneas en la raíz de cada subvolumen, volumen o conjunto +de datos (dataset). +.PP +Una forma sencilla de evitar interferencias con herramientas externas es +apuntar la configuración \`data\` a un subdirectorio normal dentro del +subvolumen o volumen (por ejemplo, \`/mnt/subvol/data\` en lugar de +\`/mnt/subvol\`). SnapRAID siempre crea el directorio de instantáneas en la +raíz del subvolumen (por ejemplo, \`/mnt/subvol/.snapraid/\`), dejando limpio +el subdirectorio de datos para que las instantáneas permanezcan fuera e +invisibles para las herramientas de copia de seguridad que operan dentro +de él. +.PP +De lo contrario, si el directorio \`data\` configurado es en sí mismo la raíz +del subvolumen o volumen, SnapRAID crea un directorio \`.snapraid/\` +directamente dentro de él (en Btrfs, Bcachefs y NTFS). Aunque SnapRAID +excluye automáticamente este directorio de su propio escaneo, las +herramientas externas de copia de seguridad o sincronización de archivos lo +encontrarán. Si no se excluye, pueden acceder a las instantáneas o enlaces +de directorio y copiar todo el árbol de datos congelado, duplicando datos y +agotando el almacenamiento de respaldo. Configure siempre las herramientas +de copia de seguridad externas para excluir el directorio \`.snapraid/\`. +.PP +En ZFS, las instantáneas se exponen a través del directorio virtual estándar +\`.zfs/snapshot/\` en lugar de \`.snapraid/\`. De forma predeterminada +(\`snapdir=hidden\`), ZFS oculta este directorio de los listados, por lo que +las herramientas externas de copia de seguridad no lo recorrerán. Si ha +habilitado \`snapdir=visible\`, asegúrese de que sus herramientas de copia +de seguridad excluyan el directorio \`.zfs/\`. .SH PATRONES (PATTERN) Los patrones proporcionan una forma flexible de filtrar archivos para su inclusión o exclusión. Al usar caracteres de globbing, puede definir reglas que coincidan con diff --git a/doc/snapraid-es.d b/doc/snapraid-es.d index 4eb03b86..beb131fd 100644 --- a/doc/snapraid-es.d +++ b/doc/snapraid-es.d @@ -1639,6 +1639,37 @@ Ciclo de vida de las instantáneas (Snapshots Lifecycle) reemplazar esa instantánea, haciendo que las versiones intermedias de los archivos no estén disponibles para la recuperación. + Copias de seguridad externas con instantáneas del sistema de archivos (External backups with filesystem snapshots) + Cuando las instantáneas del sistema de archivos están habilitadas, SnapRAID + accede o crea instantáneas en la raíz de cada subvolumen, volumen o conjunto + de datos (dataset). + + Una forma sencilla de evitar interferencias con herramientas externas es + apuntar la configuración `data` a un subdirectorio normal dentro del + subvolumen o volumen (por ejemplo, `/mnt/subvol/data` en lugar de + `/mnt/subvol`). SnapRAID siempre crea el directorio de instantáneas en la + raíz del subvolumen (por ejemplo, `/mnt/subvol/.snapraid/`), dejando limpio + el subdirectorio de datos para que las instantáneas permanezcan fuera e + invisibles para las herramientas de copia de seguridad que operan dentro + de él. + + De lo contrario, si el directorio `data` configurado es en sí mismo la raíz + del subvolumen o volumen, SnapRAID crea un directorio `.snapraid/` + directamente dentro de él (en Btrfs, Bcachefs y NTFS). Aunque SnapRAID + excluye automáticamente este directorio de su propio escaneo, las + herramientas externas de copia de seguridad o sincronización de archivos lo + encontrarán. Si no se excluye, pueden acceder a las instantáneas o enlaces + de directorio y copiar todo el árbol de datos congelado, duplicando datos y + agotando el almacenamiento de respaldo. Configure siempre las herramientas + de copia de seguridad externas para excluir el directorio `.snapraid/`. + + En ZFS, las instantáneas se exponen a través del directorio virtual estándar + `.zfs/snapshot/` en lugar de `.snapraid/`. De forma predeterminada + (`snapdir=hidden`), ZFS oculta este directorio de los listados, por lo que + las herramientas externas de copia de seguridad no lo recorrerán. Si ha + habilitado `snapdir=visible`, asegúrese de que sus herramientas de copia + de seguridad excluyan el directorio `.zfs/`. + Patrones (Pattern) Los patrones proporcionan una forma flexible de filtrar archivos para su inclusión o exclusión. Al usar caracteres de globbing, puede definir reglas que coincidan con diff --git a/doc/snapraid-es.txt b/doc/snapraid-es.txt index b4e08684..ecfd677c 100644 --- a/doc/snapraid-es.txt +++ b/doc/snapraid-es.txt @@ -1749,6 +1749,39 @@ la instantánea utilizada por el sync interrumpido. Un sync posterior puede reemplazar esa instantánea, haciendo que las versiones intermedias de los archivos no estén disponibles para la recuperación. +9.1 Copias de seguridad externas con instantáneas del sistema de archivos (External backups with filesystem snapshots) +---------------------------------------------------------------------------------------------------------------------- + +Cuando las instantáneas del sistema de archivos están habilitadas, SnapRAID +accede o crea instantáneas en la raíz de cada subvolumen, volumen o conjunto +de datos (dataset). + +Una forma sencilla de evitar interferencias con herramientas externas es +apuntar la configuración `data` a un subdirectorio normal dentro del +subvolumen o volumen (por ejemplo, `/mnt/subvol/data` en lugar de +`/mnt/subvol`). SnapRAID siempre crea el directorio de instantáneas en la +raíz del subvolumen (por ejemplo, `/mnt/subvol/.snapraid/`), dejando limpio +el subdirectorio de datos para que las instantáneas permanezcan fuera e +invisibles para las herramientas de copia de seguridad que operan dentro +de él. + +De lo contrario, si el directorio `data` configurado es en sí mismo la raíz +del subvolumen o volumen, SnapRAID crea un directorio `.snapraid/` +directamente dentro de él (en Btrfs, Bcachefs y NTFS). Aunque SnapRAID +excluye automáticamente este directorio de su propio escaneo, las +herramientas externas de copia de seguridad o sincronización de archivos lo +encontrarán. Si no se excluye, pueden acceder a las instantáneas o enlaces +de directorio y copiar todo el árbol de datos congelado, duplicando datos y +agotando el almacenamiento de respaldo. Configure siempre las herramientas +de copia de seguridad externas para excluir el directorio `.snapraid/`. + +En ZFS, las instantáneas se exponen a través del directorio virtual estándar +`.zfs/snapshot/` en lugar de `.snapraid/`. De forma predeterminada +(`snapdir=hidden`), ZFS oculta este directorio de los listados, por lo que +las herramientas externas de copia de seguridad no lo recorrerán. Si ha +habilitado `snapdir=visible`, asegúrese de que sus herramientas de copia +de seguridad excluyan el directorio `.zfs/`. + 10 PATRONES (PATTERN) ===================== diff --git a/doc/snapraid-fr.1 b/doc/snapraid-fr.1 index 03482ca6..40471034 100644 --- a/doc/snapraid-fr.1 +++ b/doc/snapraid-fr.1 @@ -2071,6 +2071,36 @@ avec succès. La parité peut encore contenir des blocs calculés à partir de l\'instantané utilisé par le sync interrompu. Un sync ultérieur peut remplacer cet instantané, rendant les versions intermédiaires des fichiers indisponibles pour la récupération. +.SS Sauvegardes externes avec les instantanés de système de fichiers (External backups with filesystem snapshots) +Lorsque les instantanés de système de fichiers sont activés, SnapRAID accède +aux instantanés ou les crée à la racine de chaque sous\-volume, volume ou +jeu de données (dataset). +.PP +Un moyen simple d\'éviter toute interférence avec les outils externes +consiste à faire pointer la configuration \`data\` vers un sous\-répertoire +ordinaire au sein du sous\-volume ou du volume (par exemple +\`/mnt/subvol/data\` au lieu de \`/mnt/subvol\`). SnapRAID crée toujours le +répertoire d\'instantanés à la racine du sous\-volume (par exemple +\`/mnt/subvol/.snapraid/\`), laissant le sous\-répertoire de données propre de +sorte que les instantanés restent à l\'extérieur et invisibles pour les +outils de sauvegarde opérant à l\'intérieur. +.PP +Sinon, si votre répertoire \`data\` configuré est lui\-même la racine du +sous\-volume ou du volume, SnapRAID crée un répertoire \`.snapraid/\` +directement à l\'intérieur (sur Btrfs, Bcachefs et NTFS). Bien que SnapRAID +exclue automatiquement ce répertoire de ses propres analyses, les outils +de sauvegarde externes ou les synchroniseurs de fichiers le rencontreront. +S\'il n\'est pas exclu, ils peuvent traverser les instantanés ou les liens de +répertoires et copier l\'ensemble de l\'arborescence figée, dupliquant les +données et risquant d\'épuiser l\'espace de sauvegarde. Configurez toujours +les outils de sauvegarde externes pour exclure le répertoire \`.snapraid/\`. +.PP +Sur ZFS, les instantanés sont exposés via le répertoire virtuel standard +\`.zfs/snapshot/\` plutôt que \`.snapraid/\`. Par défaut (\`snapdir=hidden\`), ZFS +masque ce répertoire des listes de fichiers, de sorte que les outils de +sauvegarde externes ne le parcourront pas. Si vous avez activé +\`snapdir=visible\`, veillez à ce que vos outils de sauvegarde excluent le +répertoire \`.zfs/\`. .SH MOTIF (PATTERN) Les motifs offrent un moyen flexible de filtrer les fichiers à inclure ou à exclure. En utilisant des caractères de \[dq]globbing\[dq], vous pouvez définir des règles qui diff --git a/doc/snapraid-fr.d b/doc/snapraid-fr.d index 39b5c8d1..513d0664 100644 --- a/doc/snapraid-fr.d +++ b/doc/snapraid-fr.d @@ -1676,6 +1676,37 @@ Cycle de vie des instantanés (Snapshots Lifecycle) cet instantané, rendant les versions intermédiaires des fichiers indisponibles pour la récupération. + Sauvegardes externes avec les instantanés de système de fichiers (External backups with filesystem snapshots) + Lorsque les instantanés de système de fichiers sont activés, SnapRAID accède + aux instantanés ou les crée à la racine de chaque sous-volume, volume ou + jeu de données (dataset). + + Un moyen simple d'éviter toute interférence avec les outils externes + consiste à faire pointer la configuration `data` vers un sous-répertoire + ordinaire au sein du sous-volume ou du volume (par exemple + `/mnt/subvol/data` au lieu de `/mnt/subvol`). SnapRAID crée toujours le + répertoire d'instantanés à la racine du sous-volume (par exemple + `/mnt/subvol/.snapraid/`), laissant le sous-répertoire de données propre de + sorte que les instantanés restent à l'extérieur et invisibles pour les + outils de sauvegarde opérant à l'intérieur. + + Sinon, si votre répertoire `data` configuré est lui-même la racine du + sous-volume ou du volume, SnapRAID crée un répertoire `.snapraid/` + directement à l'intérieur (sur Btrfs, Bcachefs et NTFS). Bien que SnapRAID + exclue automatiquement ce répertoire de ses propres analyses, les outils + de sauvegarde externes ou les synchroniseurs de fichiers le rencontreront. + S'il n'est pas exclu, ils peuvent traverser les instantanés ou les liens de + répertoires et copier l'ensemble de l'arborescence figée, dupliquant les + données et risquant d'épuiser l'espace de sauvegarde. Configurez toujours + les outils de sauvegarde externes pour exclure le répertoire `.snapraid/`. + + Sur ZFS, les instantanés sont exposés via le répertoire virtuel standard + `.zfs/snapshot/` plutôt que `.snapraid/`. Par défaut (`snapdir=hidden`), ZFS + masque ce répertoire des listes de fichiers, de sorte que les outils de + sauvegarde externes ne le parcourront pas. Si vous avez activé + `snapdir=visible`, veillez à ce que vos outils de sauvegarde excluent le + répertoire `.zfs/`. + Motif (Pattern) Les motifs offrent un moyen flexible de filtrer les fichiers à inclure ou à exclure. En utilisant des caractères de "globbing", vous pouvez définir des règles qui diff --git a/doc/snapraid-fr.txt b/doc/snapraid-fr.txt index 9169b1fc..c69cc2ed 100644 --- a/doc/snapraid-fr.txt +++ b/doc/snapraid-fr.txt @@ -1786,6 +1786,39 @@ l'instantané utilisé par le sync interrompu. Un sync ultérieur peut remplacer cet instantané, rendant les versions intermédiaires des fichiers indisponibles pour la récupération. +9.1 Sauvegardes externes avec les instantanés de système de fichiers (External backups with filesystem snapshots) +----------------------------------------------------------------------------------------------------------------- + +Lorsque les instantanés de système de fichiers sont activés, SnapRAID accède +aux instantanés ou les crée à la racine de chaque sous-volume, volume ou +jeu de données (dataset). + +Un moyen simple d'éviter toute interférence avec les outils externes +consiste à faire pointer la configuration `data` vers un sous-répertoire +ordinaire au sein du sous-volume ou du volume (par exemple +`/mnt/subvol/data` au lieu de `/mnt/subvol`). SnapRAID crée toujours le +répertoire d'instantanés à la racine du sous-volume (par exemple +`/mnt/subvol/.snapraid/`), laissant le sous-répertoire de données propre de +sorte que les instantanés restent à l'extérieur et invisibles pour les +outils de sauvegarde opérant à l'intérieur. + +Sinon, si votre répertoire `data` configuré est lui-même la racine du +sous-volume ou du volume, SnapRAID crée un répertoire `.snapraid/` +directement à l'intérieur (sur Btrfs, Bcachefs et NTFS). Bien que SnapRAID +exclue automatiquement ce répertoire de ses propres analyses, les outils +de sauvegarde externes ou les synchroniseurs de fichiers le rencontreront. +S'il n'est pas exclu, ils peuvent traverser les instantanés ou les liens de +répertoires et copier l'ensemble de l'arborescence figée, dupliquant les +données et risquant d'épuiser l'espace de sauvegarde. Configurez toujours +les outils de sauvegarde externes pour exclure le répertoire `.snapraid/`. + +Sur ZFS, les instantanés sont exposés via le répertoire virtuel standard +`.zfs/snapshot/` plutôt que `.snapraid/`. Par défaut (`snapdir=hidden`), ZFS +masque ce répertoire des listes de fichiers, de sorte que les outils de +sauvegarde externes ne le parcourront pas. Si vous avez activé +`snapdir=visible`, veillez à ce que vos outils de sauvegarde excluent le +répertoire `.zfs/`. + 10 MOTIF (PATTERN) ================== diff --git a/doc/snapraid-it.1 b/doc/snapraid-it.1 index ab7aff8d..5b5326df 100644 --- a/doc/snapraid-it.1 +++ b/doc/snapraid-it.1 @@ -2055,6 +2055,34 @@ con successo. La parità potrebbe ancora contenere blocchi calcolati dallo snapshot utilizzato dal sync interrotto. Un sync successivo potrebbe sostituire tale snapshot, rendendo le versioni intermedie dei file non disponibili per il recupero. +.SS Backup esterni con snapshot del filesystem (External backups with filesystem snapshots) +Quando gli snapshot del filesystem sono abilitati, SnapRAID accede o crea +gli snapshot alla radice di ciascun sottovolume, volume o dataset di dati. +.PP +Un modo semplice per evitare interferenze con gli strumenti esterni consiste +nel far puntare la configurazione \`data\` a una sottodirectory ordinaria +all\'interno del subvolume o del volume (ad esempio \`/mnt/subvol/data\` invece +di \`/mnt/subvol\`). SnapRAID crea sempre la directory degli snapshot alla +radice del subvolume (ad esempio \`/mnt/subvol/.snapraid/\`), lasciando pulita +la sottodirectory dei dati in modo che gli snapshot rimangano all\'esterno +e invisibili agli strumenti di backup che operano al suo interno. +.PP +Altrimenti, se la directory \`data\` configurata è essa stessa la radice del +subvolume o del volume, SnapRAID crea una directory \`.snapraid/\` direttamente +al suo interno (su Btrfs, Bcachefs e NTFS). Sebbene SnapRAID escluda +automaticamente questa directory dalla propria scansione, gli strumenti di +backup esterni o i programmi di sincronizzazione dei file la incontreranno. +Se non viene esclusa, potrebbero entrare negli snapshot o nei collegamenti +di directory e copiare l\'intero albero dei dati congelato, duplicando i dati +ed esaurendo lo spazio di archiviazione dei backup. Configurare sempre gli +strumenti di backup esterni in modo da escludere la directory \`.snapraid/\`. +.PP +Su ZFS, gli snapshot sono esposti tramite la directory virtuale standard +\`.zfs/snapshot/\` anziché \`.snapraid/\`. Di default (\`snapdir=hidden\`), ZFS +nasconde questa directory dall\'elenco delle directory, quindi gli strumenti +di backup esterni non la attraverseranno. Se è stato abilitato +\`snapdir=visible\`, assicurarsi che gli strumenti di backup escludano la +directory \`.zfs/\`. .SH PATTERN I pattern offrono un modo flessibile per filtrare i file da includere o escludere. Usando i caratteri globbing, è possibile definire regole che diff --git a/doc/snapraid-it.d b/doc/snapraid-it.d index a300db88..e69417a5 100644 --- a/doc/snapraid-it.d +++ b/doc/snapraid-it.d @@ -1649,6 +1649,35 @@ Ciclo di vita degli snapshot (Snapshots Lifecycle) tale snapshot, rendendo le versioni intermedie dei file non disponibili per il recupero. + Backup esterni con snapshot del filesystem (External backups with filesystem snapshots) + Quando gli snapshot del filesystem sono abilitati, SnapRAID accede o crea + gli snapshot alla radice di ciascun sottovolume, volume o dataset di dati. + + Un modo semplice per evitare interferenze con gli strumenti esterni consiste + nel far puntare la configurazione `data` a una sottodirectory ordinaria + all'interno del subvolume o del volume (ad esempio `/mnt/subvol/data` invece + di `/mnt/subvol`). SnapRAID crea sempre la directory degli snapshot alla + radice del subvolume (ad esempio `/mnt/subvol/.snapraid/`), lasciando pulita + la sottodirectory dei dati in modo che gli snapshot rimangano all'esterno + e invisibili agli strumenti di backup che operano al suo interno. + + Altrimenti, se la directory `data` configurata è essa stessa la radice del + subvolume o del volume, SnapRAID crea una directory `.snapraid/` direttamente + al suo interno (su Btrfs, Bcachefs e NTFS). Sebbene SnapRAID escluda + automaticamente questa directory dalla propria scansione, gli strumenti di + backup esterni o i programmi di sincronizzazione dei file la incontreranno. + Se non viene esclusa, potrebbero entrare negli snapshot o nei collegamenti + di directory e copiare l'intero albero dei dati congelato, duplicando i dati + ed esaurendo lo spazio di archiviazione dei backup. Configurare sempre gli + strumenti di backup esterni in modo da escludere la directory `.snapraid/`. + + Su ZFS, gli snapshot sono esposti tramite la directory virtuale standard + `.zfs/snapshot/` anziché `.snapraid/`. Di default (`snapdir=hidden`), ZFS + nasconde questa directory dall'elenco delle directory, quindi gli strumenti + di backup esterni non la attraverseranno. Se è stato abilitato + `snapdir=visible`, assicurarsi che gli strumenti di backup escludano la + directory `.zfs/`. + Pattern I pattern offrono un modo flessibile per filtrare i file da includere o escludere. Usando i caratteri globbing, è possibile definire regole che diff --git a/doc/snapraid-it.txt b/doc/snapraid-it.txt index bf0b3c36..711a98cf 100644 --- a/doc/snapraid-it.txt +++ b/doc/snapraid-it.txt @@ -1759,6 +1759,37 @@ snapshot utilizzato dal sync interrotto. Un sync successivo potrebbe sostituire tale snapshot, rendendo le versioni intermedie dei file non disponibili per il recupero. +9.1 Backup esterni con snapshot del filesystem (External backups with filesystem snapshots) +------------------------------------------------------------------------------------------- + +Quando gli snapshot del filesystem sono abilitati, SnapRAID accede o crea +gli snapshot alla radice di ciascun sottovolume, volume o dataset di dati. + +Un modo semplice per evitare interferenze con gli strumenti esterni consiste +nel far puntare la configurazione `data` a una sottodirectory ordinaria +all'interno del subvolume o del volume (ad esempio `/mnt/subvol/data` invece +di `/mnt/subvol`). SnapRAID crea sempre la directory degli snapshot alla +radice del subvolume (ad esempio `/mnt/subvol/.snapraid/`), lasciando pulita +la sottodirectory dei dati in modo che gli snapshot rimangano all'esterno +e invisibili agli strumenti di backup che operano al suo interno. + +Altrimenti, se la directory `data` configurata è essa stessa la radice del +subvolume o del volume, SnapRAID crea una directory `.snapraid/` direttamente +al suo interno (su Btrfs, Bcachefs e NTFS). Sebbene SnapRAID escluda +automaticamente questa directory dalla propria scansione, gli strumenti di +backup esterni o i programmi di sincronizzazione dei file la incontreranno. +Se non viene esclusa, potrebbero entrare negli snapshot o nei collegamenti +di directory e copiare l'intero albero dei dati congelato, duplicando i dati +ed esaurendo lo spazio di archiviazione dei backup. Configurare sempre gli +strumenti di backup esterni in modo da escludere la directory `.snapraid/`. + +Su ZFS, gli snapshot sono esposti tramite la directory virtuale standard +`.zfs/snapshot/` anziché `.snapraid/`. Di default (`snapdir=hidden`), ZFS +nasconde questa directory dall'elenco delle directory, quindi gli strumenti +di backup esterni non la attraverseranno. Se è stato abilitato +`snapdir=visible`, assicurarsi che gli strumenti di backup escludano la +directory `.zfs/`. + 10 PATTERN ========== diff --git a/doc/snapraid-ja.1 b/doc/snapraid-ja.1 index efdca0e8..a11a9d20 100644 --- a/doc/snapraid-ja.1 +++ b/doc/snapraid-ja.1 @@ -1833,6 +1833,31 @@ ZFSでは標準の \`.zfs/snapshot/\` が使用されます。 パリティには、中断された sync で使用されたスナップショットから計算されたブロックが まだ含まれている可能性があります。その後の sync によりそのスナップショットが 置き換えられ、中間バージョンのファイルが復元できなくなる可能性があります。 +.SS ファイルシステムスナップショットと外部バックアップ (External backups with filesystem snapshots) +ファイルシステムスナップショットが有効な場合、SnapRAIDは各データサブボリューム、 +ボリューム、またはデータセットのルートにあるスナップショットにアクセスまたは作成します。 +.PP +外部ツールとの干渉を避ける簡単な方法は、\`data\` 設定をサブボリュームまたは +ボリューム内の通常のサブディレクトリ(例えば \`/mnt/subvol\` の代わりに +\`/mnt/subvol/data\`)に指定することです。SnapRAIDは常にサブボリュームのルートに +スナップショットディレクトリを作成するため(例えば \`/mnt/subvol/.snapraid/\`)、 +データサブディレクトリがクリーンに保たれ、スナップショットはそのディレクトリ内で +動作するバックアップツールから見えない状態のままになります。 +.PP +一方、設定された \`data\` ディレクトリ自体がサブボリュームまたはボリュームの +ルートである場合、SnapRAIDはその直下に \`.snapraid/\` ディレクトリを作成します(Btrfs、 +Bcachefs、およびNTFS)。SnapRAID自身のスキャンからは自動的に除外されますが、 +外部のバックアップツールやファイル同期ツールはそのディレクトリを検出します。 +除外設定を行わない場合、スナップショットやディレクトリリンク内を走査して凍結された +データツリー全体をコピーしてしまい、データが重複してバックアップストレージを使い果たす +可能性があります。外部バックアップツールでは、必ず \`.snapraid/\` ディレクトリを +除外するように設定してください。 +.PP +ZFSでは、スナップショットは \`.snapraid/\` ではなく標準の仮想ディレクトリ +\`.zfs/snapshot/\` 経由で公開されます。デフォルト(\`snapdir=hidden\`)では、ZFSはこの +ディレクトリを一覧表示から隠すため、外部バックアップツールが走査することはありません。 +\`snapdir=visible\` を有効にしている場合は、バックアップツールが \`.zfs/\` ディレクトリを +除外するように設定してください。 .SH パターン (PATTERN) パターンは、含めるファイルまたは除外するファイルをフィルタリングするための柔軟な方法を 提供します。グロブ文字を使用することで、すべてのパスを手動でリストすることなく、 diff --git a/doc/snapraid-ja.d b/doc/snapraid-ja.d index 3931aefe..58a996b7 100644 --- a/doc/snapraid-ja.d +++ b/doc/snapraid-ja.d @@ -1438,6 +1438,32 @@ Name{number} まだ含まれている可能性があります。その後の sync によりそのスナップショットが 置き換えられ、中間バージョンのファイルが復元できなくなる可能性があります。 + ファイルシステムスナップショットと外部バックアップ (External backups with filesystem snapshots) + ファイルシステムスナップショットが有効な場合、SnapRAIDは各データサブボリューム、 + ボリューム、またはデータセットのルートにあるスナップショットにアクセスまたは作成します。 + + 外部ツールとの干渉を避ける簡単な方法は、`data` 設定をサブボリュームまたは + ボリューム内の通常のサブディレクトリ(例えば `/mnt/subvol` の代わりに + `/mnt/subvol/data`)に指定することです。SnapRAIDは常にサブボリュームのルートに + スナップショットディレクトリを作成するため(例えば `/mnt/subvol/.snapraid/`)、 + データサブディレクトリがクリーンに保たれ、スナップショットはそのディレクトリ内で + 動作するバックアップツールから見えない状態のままになります。 + + 一方、設定された `data` ディレクトリ自体がサブボリュームまたはボリュームの + ルートである場合、SnapRAIDはその直下に `.snapraid/` ディレクトリを作成します(Btrfs、 + Bcachefs、およびNTFS)。SnapRAID自身のスキャンからは自動的に除外されますが、 + 外部のバックアップツールやファイル同期ツールはそのディレクトリを検出します。 + 除外設定を行わない場合、スナップショットやディレクトリリンク内を走査して凍結された + データツリー全体をコピーしてしまい、データが重複してバックアップストレージを使い果たす + 可能性があります。外部バックアップツールでは、必ず `.snapraid/` ディレクトリを + 除外するように設定してください。 + + ZFSでは、スナップショットは `.snapraid/` ではなく標準の仮想ディレクトリ + `.zfs/snapshot/` 経由で公開されます。デフォルト(`snapdir=hidden`)では、ZFSはこの + ディレクトリを一覧表示から隠すため、外部バックアップツールが走査することはありません。 + `snapdir=visible` を有効にしている場合は、バックアップツールが `.zfs/` ディレクトリを + 除外するように設定してください。 + パターン (Pattern) パターンは、含めるファイルまたは除外するファイルをフィルタリングするための柔軟な方法を 提供します。グロブ文字を使用することで、すべてのパスを手動でリストすることなく、 diff --git a/doc/snapraid-ja.txt b/doc/snapraid-ja.txt index f8b28a74..580201d5 100644 --- a/doc/snapraid-ja.txt +++ b/doc/snapraid-ja.txt @@ -1548,6 +1548,34 @@ ZFSでは標準の `.zfs/snapshot/` が使用されます。 まだ含まれている可能性があります。その後の sync によりそのスナップショットが 置き換えられ、中間バージョンのファイルが復元できなくなる可能性があります。 +9.1 ファイルシステムスナップショットと外部バックアップ (External backups with filesystem snapshots) +-------------------------------------------------------------------------- + +ファイルシステムスナップショットが有効な場合、SnapRAIDは各データサブボリューム、 +ボリューム、またはデータセットのルートにあるスナップショットにアクセスまたは作成します。 + +外部ツールとの干渉を避ける簡単な方法は、`data` 設定をサブボリュームまたは +ボリューム内の通常のサブディレクトリ(例えば `/mnt/subvol` の代わりに +`/mnt/subvol/data`)に指定することです。SnapRAIDは常にサブボリュームのルートに +スナップショットディレクトリを作成するため(例えば `/mnt/subvol/.snapraid/`)、 +データサブディレクトリがクリーンに保たれ、スナップショットはそのディレクトリ内で +動作するバックアップツールから見えない状態のままになります。 + +一方、設定された `data` ディレクトリ自体がサブボリュームまたはボリュームの +ルートである場合、SnapRAIDはその直下に `.snapraid/` ディレクトリを作成します(Btrfs、 +Bcachefs、およびNTFS)。SnapRAID自身のスキャンからは自動的に除外されますが、 +外部のバックアップツールやファイル同期ツールはそのディレクトリを検出します。 +除外設定を行わない場合、スナップショットやディレクトリリンク内を走査して凍結された +データツリー全体をコピーしてしまい、データが重複してバックアップストレージを使い果たす +可能性があります。外部バックアップツールでは、必ず `.snapraid/` ディレクトリを +除外するように設定してください。 + +ZFSでは、スナップショットは `.snapraid/` ではなく標準の仮想ディレクトリ +`.zfs/snapshot/` 経由で公開されます。デフォルト(`snapdir=hidden`)では、ZFSはこの +ディレクトリを一覧表示から隠すため、外部バックアップツールが走査することはありません。 +`snapdir=visible` を有効にしている場合は、バックアップツールが `.zfs/` ディレクトリを +除外するように設定してください。 + 10 パターン (PATTERN) ================= diff --git a/doc/snapraid-ko.1 b/doc/snapraid-ko.1 index c5af4752..406369b0 100644 --- a/doc/snapraid-ko.1 +++ b/doc/snapraid-ko.1 @@ -1877,6 +1877,28 @@ ZFS에서는 표준 \`.zfs/snapshot/\` 이 사용됩니다. 패리티에는 중단된 sync에서 사용된 스냅샷에서 계산된 블록이 여전히 포함되어 있을 수 있습니다. 이후의 sync로 인해 해당 스냅샷이 교체되어 복구에 필요한 중간 버전의 파일을 사용할 수 없게 될 수 있습니다. +.SS 파일 시스템 스냅샷을 사용한 외부 백업 (External backups with filesystem snapshots) +파일 시스템 스냅샷이 활성화되면 SnapRAID는 각 데이터 서브볼륨, 볼륨 또는 +데이터셋의 루트에서 스냅샷에 접근하거나 생성합니다. +.PP +외부 도구와의 간섭을 피하는 간단한 방법은 서브볼륨이나 볼륨 내의 일반 +하위 디렉터리(예: \`/mnt/subvol\` 대신 \`/mnt/subvol/data\`)를 가리키도록 +\`data\` 를 설정하는 것입니다. SnapRAID는 항상 서브볼륨 루트에 스냅샷 디렉터리를 +생성하므로(예: \`/mnt/subvol/.snapraid/\`), 데이터 하위 디렉터리를 깨끗하게 +유지하여 해당 디렉터리 내에서 작동하는 백업 도구에 스냅샷이 노출되지 않도록 합니다. +.PP +그렇지 않고 설정된 \`data\` 디렉터리 자체가 서브볼륨이나 볼륨 루트인 경우, +SnapRAID는 해당 디렉터리 바로 안에 \`.snapraid/\` 디렉터리를 생성합니다(Btrfs, +Bcachefs 및 NTFS). SnapRAID 자체 스캔에서는 이 디렉터리가 자동으로 제외되지만, +외부 백업 도구나 파일 동기화 도구는 이를 발견하게 됩니다. 제외하지 않으면 +스냅샷이나 디렉터리 링크를 탐색하여 동결된 전체 데이터 트리를 복사함으로써 +데이터를 중복 생성하고 백업 저장 공간을 고갈시킬 수 있습니다. 항상 외부 백업 +도구에서 \`.snapraid/\` 디렉터리를 제외하도록 구성하십시오. +.PP +ZFS에서는 스냅샷이 \`.snapraid/\` 대신 표준 가상 디렉터리인 \`.zfs/snapshot/\` 을 +통해 노출됩니다. 기본값(\`snapdir=hidden\`)에서는 ZFS가 디렉터리 목록에서 이 +디렉터리를 숨기므로 외부 백업 도구가 이를 탐색하지 않습니다. \`snapdir=visible\` 로 +설정한 경우 백업 도구에서 \`.zfs/\` 디렉터리를 제외하도록 하십시오. .SH 패턴 (PATTERN) 패턴은 포함하거나 제외할 파일을 필터링하는 유연한 방법을 제공합니다. 글로빙(globbing) 문자를 사용하면 모든 경로를 일일이 나열하지 않고도 diff --git a/doc/snapraid-ko.d b/doc/snapraid-ko.d index 896fc3f7..181827b1 100644 --- a/doc/snapraid-ko.d +++ b/doc/snapraid-ko.d @@ -1482,6 +1482,29 @@ Name{number} 여전히 포함되어 있을 수 있습니다. 이후의 sync로 인해 해당 스냅샷이 교체되어 복구에 필요한 중간 버전의 파일을 사용할 수 없게 될 수 있습니다. + 파일 시스템 스냅샷을 사용한 외부 백업 (External backups with filesystem snapshots) + 파일 시스템 스냅샷이 활성화되면 SnapRAID는 각 데이터 서브볼륨, 볼륨 또는 + 데이터셋의 루트에서 스냅샷에 접근하거나 생성합니다. + + 외부 도구와의 간섭을 피하는 간단한 방법은 서브볼륨이나 볼륨 내의 일반 + 하위 디렉터리(예: `/mnt/subvol` 대신 `/mnt/subvol/data`)를 가리키도록 + `data` 를 설정하는 것입니다. SnapRAID는 항상 서브볼륨 루트에 스냅샷 디렉터리를 + 생성하므로(예: `/mnt/subvol/.snapraid/`), 데이터 하위 디렉터리를 깨끗하게 + 유지하여 해당 디렉터리 내에서 작동하는 백업 도구에 스냅샷이 노출되지 않도록 합니다. + + 그렇지 않고 설정된 `data` 디렉터리 자체가 서브볼륨이나 볼륨 루트인 경우, + SnapRAID는 해당 디렉터리 바로 안에 `.snapraid/` 디렉터리를 생성합니다(Btrfs, + Bcachefs 및 NTFS). SnapRAID 자체 스캔에서는 이 디렉터리가 자동으로 제외되지만, + 외부 백업 도구나 파일 동기화 도구는 이를 발견하게 됩니다. 제외하지 않으면 + 스냅샷이나 디렉터리 링크를 탐색하여 동결된 전체 데이터 트리를 복사함으로써 + 데이터를 중복 생성하고 백업 저장 공간을 고갈시킬 수 있습니다. 항상 외부 백업 + 도구에서 `.snapraid/` 디렉터리를 제외하도록 구성하십시오. + + ZFS에서는 스냅샷이 `.snapraid/` 대신 표준 가상 디렉터리인 `.zfs/snapshot/` 을 + 통해 노출됩니다. 기본값(`snapdir=hidden`)에서는 ZFS가 디렉터리 목록에서 이 + 디렉터리를 숨기므로 외부 백업 도구가 이를 탐색하지 않습니다. `snapdir=visible` 로 + 설정한 경우 백업 도구에서 `.zfs/` 디렉터리를 제외하도록 하십시오. + 패턴 (Pattern) 패턴은 포함하거나 제외할 파일을 필터링하는 유연한 방법을 제공합니다. 글로빙(globbing) 문자를 사용하면 모든 경로를 일일이 나열하지 않고도 diff --git a/doc/snapraid-ko.txt b/doc/snapraid-ko.txt index db1e385d..0bc5dd8b 100644 --- a/doc/snapraid-ko.txt +++ b/doc/snapraid-ko.txt @@ -1592,6 +1592,31 @@ ZFS에서는 표준 `.zfs/snapshot/` 이 사용됩니다. 여전히 포함되어 있을 수 있습니다. 이후의 sync로 인해 해당 스냅샷이 교체되어 복구에 필요한 중간 버전의 파일을 사용할 수 없게 될 수 있습니다. +9.1 파일 시스템 스냅샷을 사용한 외부 백업 (External backups with filesystem snapshots) +---------------------------------------------------------------------- + +파일 시스템 스냅샷이 활성화되면 SnapRAID는 각 데이터 서브볼륨, 볼륨 또는 +데이터셋의 루트에서 스냅샷에 접근하거나 생성합니다. + +외부 도구와의 간섭을 피하는 간단한 방법은 서브볼륨이나 볼륨 내의 일반 +하위 디렉터리(예: `/mnt/subvol` 대신 `/mnt/subvol/data`)를 가리키도록 +`data` 를 설정하는 것입니다. SnapRAID는 항상 서브볼륨 루트에 스냅샷 디렉터리를 +생성하므로(예: `/mnt/subvol/.snapraid/`), 데이터 하위 디렉터리를 깨끗하게 +유지하여 해당 디렉터리 내에서 작동하는 백업 도구에 스냅샷이 노출되지 않도록 합니다. + +그렇지 않고 설정된 `data` 디렉터리 자체가 서브볼륨이나 볼륨 루트인 경우, +SnapRAID는 해당 디렉터리 바로 안에 `.snapraid/` 디렉터리를 생성합니다(Btrfs, +Bcachefs 및 NTFS). SnapRAID 자체 스캔에서는 이 디렉터리가 자동으로 제외되지만, +외부 백업 도구나 파일 동기화 도구는 이를 발견하게 됩니다. 제외하지 않으면 +스냅샷이나 디렉터리 링크를 탐색하여 동결된 전체 데이터 트리를 복사함으로써 +데이터를 중복 생성하고 백업 저장 공간을 고갈시킬 수 있습니다. 항상 외부 백업 +도구에서 `.snapraid/` 디렉터리를 제외하도록 구성하십시오. + +ZFS에서는 스냅샷이 `.snapraid/` 대신 표준 가상 디렉터리인 `.zfs/snapshot/` 을 +통해 노출됩니다. 기본값(`snapdir=hidden`)에서는 ZFS가 디렉터리 목록에서 이 +디렉터리를 숨기므로 외부 백업 도구가 이를 탐색하지 않습니다. `snapdir=visible` 로 +설정한 경우 백업 도구에서 `.zfs/` 디렉터리를 제외하도록 하십시오. + 10 패턴 (PATTERN) =============== diff --git a/doc/snapraid-pl.1 b/doc/snapraid-pl.1 index be4409c7..f58b449d 100644 --- a/doc/snapraid-pl.1 +++ b/doc/snapraid-pl.1 @@ -1954,6 +1954,36 @@ usuwania, zastępowania lub przenoszenia plików do momentu pomyślnego ukończe kolejnej operacji \`sync\`. Parzystość może nadal zawierać bloki obliczone na podstawie migawki użytej przez przerwaną synchronizację. Późniejsza synchronizacja może zastąpić tę migawkę, uniemożliwiając odzyskanie pośrednich wersji plików. +.SS Kopie zapasowe z migawkami systemu plików (External backups with filesystem snapshots) +Gdy migawki systemu plików są włączone, SnapRAID uzyskuje dostęp do migawek +lub tworzy je w katalogu głównym każdego podwoluminu danych, woluminu lub +zestawu danych (dataset). +.PP +Prostym sposobem na uniknięcie jakichkolwiek konfliktów z zewnętrznymi +narzędziami jest wskazanie w konfiguracji \`data\` zwykłego podkatalogu +wewnątrz podwoluminu lub woluminu (na przykład \`/mnt/subvol/data\` zamiast +\`/mnt/subvol\`). SnapRAID zawsze tworzy katalog migawek w katalogu głównym +podwoluminu (na przykład \`/mnt/subvol/.snapraid/\`), pozostawiając podkatalog +danych czysty, dzięki czemu migawki pozostają na zewnątrz i są niewidoczne +dla narzędzi do tworzenia kopii zapasowych działających w tym katalogu. +.PP +W przeciwnym razie, jeśli skonfigurowany katalog \`data\` jest sam w sobie +katalogiem głównym podwoluminu lub woluminu, SnapRAID utworzy katalog +\`.snapraid/\` bezpośrednio w nim (w systemach Btrfs, Bcachefs i NTFS). +Chociaż SnapRAID automatycznie wyklucza ten katalog z własnego skanowania, +zewnętrzne narzędzia do tworzenia kopii zapasowych lub synchronizacji +plików napotkają go. Jeśli nie zostanie wykluczony, mogą wejść do migawek +lub dowiązań katalogów i skopiować całe zamrożone drzewo danych, duplikując +dane i wyczerpując przestrzeń dyskową kopii zapasowej. Zawsze konfiguruj +zewnętrzne narzędzia do tworzenia kopii zapasowych tak, aby wykluczały +katalog \`.snapraid/\`. +.PP +W systemie ZFS migawki są udostępniane za pośrednictwem standardowego +katalogu wirtualnego \`.zfs/snapshot/\` zamiast \`.snapraid/\`. Domyślnie +(\`snapdir=hidden\`) system ZFS ukrywa ten katalog na listach katalogów, więc +zewnętrzne narzędzia do tworzenia kopii zapasowych go nie przeskanują. Jeśli +włączono opcję \`snapdir=visible\`, upewnij się, że narzędzia do tworzenia kopii +zapasowych wykluczają katalog \`.zfs/\`. .SH WZORCE (PATTERN) Wzorce zapewniają elastyczny sposób filtrowania plików do uwzględnienia lub wykluczenia. Używając znaków globbingowych, można zdefiniować reguły, które diff --git a/doc/snapraid-pl.d b/doc/snapraid-pl.d index 164a97ab..5994af30 100644 --- a/doc/snapraid-pl.d +++ b/doc/snapraid-pl.d @@ -1559,6 +1559,37 @@ Cykl życia migawek (Snapshots Lifecycle) migawki użytej przez przerwaną synchronizację. Późniejsza synchronizacja może zastąpić tę migawkę, uniemożliwiając odzyskanie pośrednich wersji plików. + Kopie zapasowe z migawkami systemu plików (External backups with filesystem snapshots) + Gdy migawki systemu plików są włączone, SnapRAID uzyskuje dostęp do migawek + lub tworzy je w katalogu głównym każdego podwoluminu danych, woluminu lub + zestawu danych (dataset). + + Prostym sposobem na uniknięcie jakichkolwiek konfliktów z zewnętrznymi + narzędziami jest wskazanie w konfiguracji `data` zwykłego podkatalogu + wewnątrz podwoluminu lub woluminu (na przykład `/mnt/subvol/data` zamiast + `/mnt/subvol`). SnapRAID zawsze tworzy katalog migawek w katalogu głównym + podwoluminu (na przykład `/mnt/subvol/.snapraid/`), pozostawiając podkatalog + danych czysty, dzięki czemu migawki pozostają na zewnątrz i są niewidoczne + dla narzędzi do tworzenia kopii zapasowych działających w tym katalogu. + + W przeciwnym razie, jeśli skonfigurowany katalog `data` jest sam w sobie + katalogiem głównym podwoluminu lub woluminu, SnapRAID utworzy katalog + `.snapraid/` bezpośrednio w nim (w systemach Btrfs, Bcachefs i NTFS). + Chociaż SnapRAID automatycznie wyklucza ten katalog z własnego skanowania, + zewnętrzne narzędzia do tworzenia kopii zapasowych lub synchronizacji + plików napotkają go. Jeśli nie zostanie wykluczony, mogą wejść do migawek + lub dowiązań katalogów i skopiować całe zamrożone drzewo danych, duplikując + dane i wyczerpując przestrzeń dyskową kopii zapasowej. Zawsze konfiguruj + zewnętrzne narzędzia do tworzenia kopii zapasowych tak, aby wykluczały + katalog `.snapraid/`. + + W systemie ZFS migawki są udostępniane za pośrednictwem standardowego + katalogu wirtualnego `.zfs/snapshot/` zamiast `.snapraid/`. Domyślnie + (`snapdir=hidden`) system ZFS ukrywa ten katalog na listach katalogów, więc + zewnętrzne narzędzia do tworzenia kopii zapasowych go nie przeskanują. Jeśli + włączono opcję `snapdir=visible`, upewnij się, że narzędzia do tworzenia kopii + zapasowych wykluczają katalog `.zfs/`. + Wzorce (Pattern) Wzorce zapewniają elastyczny sposób filtrowania plików do uwzględnienia lub wykluczenia. Używając znaków globbingowych, można zdefiniować reguły, które diff --git a/doc/snapraid-pl.txt b/doc/snapraid-pl.txt index 8b7af044..8ad42b3c 100644 --- a/doc/snapraid-pl.txt +++ b/doc/snapraid-pl.txt @@ -1669,6 +1669,39 @@ kolejnej operacji `sync`. Parzystość może nadal zawierać bloki obliczone na migawki użytej przez przerwaną synchronizację. Późniejsza synchronizacja może zastąpić tę migawkę, uniemożliwiając odzyskanie pośrednich wersji plików. +9.1 Kopie zapasowe z migawkami systemu plików (External backups with filesystem snapshots) +------------------------------------------------------------------------------------------ + +Gdy migawki systemu plików są włączone, SnapRAID uzyskuje dostęp do migawek +lub tworzy je w katalogu głównym każdego podwoluminu danych, woluminu lub +zestawu danych (dataset). + +Prostym sposobem na uniknięcie jakichkolwiek konfliktów z zewnętrznymi +narzędziami jest wskazanie w konfiguracji `data` zwykłego podkatalogu +wewnątrz podwoluminu lub woluminu (na przykład `/mnt/subvol/data` zamiast +`/mnt/subvol`). SnapRAID zawsze tworzy katalog migawek w katalogu głównym +podwoluminu (na przykład `/mnt/subvol/.snapraid/`), pozostawiając podkatalog +danych czysty, dzięki czemu migawki pozostają na zewnątrz i są niewidoczne +dla narzędzi do tworzenia kopii zapasowych działających w tym katalogu. + +W przeciwnym razie, jeśli skonfigurowany katalog `data` jest sam w sobie +katalogiem głównym podwoluminu lub woluminu, SnapRAID utworzy katalog +`.snapraid/` bezpośrednio w nim (w systemach Btrfs, Bcachefs i NTFS). +Chociaż SnapRAID automatycznie wyklucza ten katalog z własnego skanowania, +zewnętrzne narzędzia do tworzenia kopii zapasowych lub synchronizacji +plików napotkają go. Jeśli nie zostanie wykluczony, mogą wejść do migawek +lub dowiązań katalogów i skopiować całe zamrożone drzewo danych, duplikując +dane i wyczerpując przestrzeń dyskową kopii zapasowej. Zawsze konfiguruj +zewnętrzne narzędzia do tworzenia kopii zapasowych tak, aby wykluczały +katalog `.snapraid/`. + +W systemie ZFS migawki są udostępniane za pośrednictwem standardowego +katalogu wirtualnego `.zfs/snapshot/` zamiast `.snapraid/`. Domyślnie +(`snapdir=hidden`) system ZFS ukrywa ten katalog na listach katalogów, więc +zewnętrzne narzędzia do tworzenia kopii zapasowych go nie przeskanują. Jeśli +włączono opcję `snapdir=visible`, upewnij się, że narzędzia do tworzenia kopii +zapasowych wykluczają katalog `.zfs/`. + 10 WZORCE (PATTERN) =================== diff --git a/doc/snapraid-pt.1 b/doc/snapraid-pt.1 index 08ad589a..42c6fa53 100644 --- a/doc/snapraid-pt.1 +++ b/doc/snapraid-pt.1 @@ -1992,6 +1992,36 @@ com sucesso. A paridade ainda pode conter blocos calculados a partir do instantâneo usado pelo sync interrompido. Um sync posterior pode substituir esse instantâneo, tornando as versões intermediárias dos arquivos indisponíveis para recuperação. +.SS Cópias de segurança externas com instantâneos do sistema de ficheiros (External backups with filesystem snapshots) +Quando os instantâneos do sistema de ficheiros estão ativados, o SnapRAID +acede ou cria instantâneos na raiz de cada subvolume, volume ou conjunto +de dados (dataset). +.PP +Uma forma simples de evitar qualquer interferência com ferramentas externas +é apontar a configuração \`data\` para um subdiretório normal dentro do +subvolume ou volume (por exemplo, \`/mnt/subvol/data\` em vez de \`/mnt/subvol\`). +O SnapRAID cria sempre o diretório de instantâneos na raiz do subvolume +(por exemplo, \`/mnt/subvol/.snapraid/\`), deixando o subdiretório de dados +limpo, de modo que os instantâneos permaneçam fora e invisíveis para as +ferramentas de cópia de segurança que operam dentro dele. +.PP +Caso contrário, se o diretório \`data\` configurado for ele próprio a raiz +do subvolume ou do volume, o SnapRAID cria um diretório \`.snapraid/\` +diretamente dentro dele (em Btrfs, Bcachefs e NTFS). Embora o SnapRAID +exclua automaticamente esse diretório das suas próprias verificações, +ferramentas externas de cópia de segurança ou de sincronização irão +encontrá\-lo. Se não for excluído, podem atravessar os instantâneos ou +ligações de diretório e copiar toda a árvore de dados congelada, duplicando +dados e esgotando o armazenamento de backup. Configure sempre as +ferramentas de cópia de segurança externas para excluir o diretório +\`.snapraid/\`. +.PP +No ZFS, os instantâneos são expostos através do diretório virtual padrão +\`.zfs/snapshot/\` em vez de \`.snapraid/\`. Por predefinição (\`snapdir=hidden\`), +o ZFS oculta esse diretório das listagens, de modo que ferramentas externas +de cópia de segurança não o percorrem. Se tiver ativado \`snapdir=visible\`, +certifique\-se de que as suas ferramentas de cópia de segurança excluem o +diretório \`.zfs/\`. .SH PADRãO (PATTERN) Os padrões fornecem uma maneira flexível de filtrar arquivos para inclusão ou exclusão. Ao usar caracteres de globbing, você pode definir regras que diff --git a/doc/snapraid-pt.d b/doc/snapraid-pt.d index e18a4cbd..029e06b1 100644 --- a/doc/snapraid-pt.d +++ b/doc/snapraid-pt.d @@ -1597,6 +1597,37 @@ Ciclo de Vida dos Instantâneos (Snapshots Lifecycle) esse instantâneo, tornando as versões intermediárias dos arquivos indisponíveis para recuperação. + Cópias de segurança externas com instantâneos do sistema de ficheiros (External backups with filesystem snapshots) + Quando os instantâneos do sistema de ficheiros estão ativados, o SnapRAID + acede ou cria instantâneos na raiz de cada subvolume, volume ou conjunto + de dados (dataset). + + Uma forma simples de evitar qualquer interferência com ferramentas externas + é apontar a configuração `data` para um subdiretório normal dentro do + subvolume ou volume (por exemplo, `/mnt/subvol/data` em vez de `/mnt/subvol`). + O SnapRAID cria sempre o diretório de instantâneos na raiz do subvolume + (por exemplo, `/mnt/subvol/.snapraid/`), deixando o subdiretório de dados + limpo, de modo que os instantâneos permaneçam fora e invisíveis para as + ferramentas de cópia de segurança que operam dentro dele. + + Caso contrário, se o diretório `data` configurado for ele próprio a raiz + do subvolume ou do volume, o SnapRAID cria um diretório `.snapraid/` + diretamente dentro dele (em Btrfs, Bcachefs e NTFS). Embora o SnapRAID + exclua automaticamente esse diretório das suas próprias verificações, + ferramentas externas de cópia de segurança ou de sincronização irão + encontrá-lo. Se não for excluído, podem atravessar os instantâneos ou + ligações de diretório e copiar toda a árvore de dados congelada, duplicando + dados e esgotando o armazenamento de backup. Configure sempre as + ferramentas de cópia de segurança externas para excluir o diretório + `.snapraid/`. + + No ZFS, os instantâneos são expostos através do diretório virtual padrão + `.zfs/snapshot/` em vez de `.snapraid/`. Por predefinição (`snapdir=hidden`), + o ZFS oculta esse diretório das listagens, de modo que ferramentas externas + de cópia de segurança não o percorrem. Se tiver ativado `snapdir=visible`, + certifique-se de que as suas ferramentas de cópia de segurança excluem o + diretório `.zfs/`. + Padrão (Pattern) Os padrões fornecem uma maneira flexível de filtrar arquivos para inclusão ou exclusão. Ao usar caracteres de globbing, você pode definir regras que diff --git a/doc/snapraid-pt.txt b/doc/snapraid-pt.txt index a6d6edb4..654c4c93 100644 --- a/doc/snapraid-pt.txt +++ b/doc/snapraid-pt.txt @@ -1707,6 +1707,39 @@ instantâneo usado pelo sync interrompido. Um sync posterior pode substituir esse instantâneo, tornando as versões intermediárias dos arquivos indisponíveis para recuperação. +9.1 Cópias de segurança externas com instantâneos do sistema de ficheiros (External backups with filesystem snapshots) +---------------------------------------------------------------------------------------------------------------------- + +Quando os instantâneos do sistema de ficheiros estão ativados, o SnapRAID +acede ou cria instantâneos na raiz de cada subvolume, volume ou conjunto +de dados (dataset). + +Uma forma simples de evitar qualquer interferência com ferramentas externas +é apontar a configuração `data` para um subdiretório normal dentro do +subvolume ou volume (por exemplo, `/mnt/subvol/data` em vez de `/mnt/subvol`). +O SnapRAID cria sempre o diretório de instantâneos na raiz do subvolume +(por exemplo, `/mnt/subvol/.snapraid/`), deixando o subdiretório de dados +limpo, de modo que os instantâneos permaneçam fora e invisíveis para as +ferramentas de cópia de segurança que operam dentro dele. + +Caso contrário, se o diretório `data` configurado for ele próprio a raiz +do subvolume ou do volume, o SnapRAID cria um diretório `.snapraid/` +diretamente dentro dele (em Btrfs, Bcachefs e NTFS). Embora o SnapRAID +exclua automaticamente esse diretório das suas próprias verificações, +ferramentas externas de cópia de segurança ou de sincronização irão +encontrá-lo. Se não for excluído, podem atravessar os instantâneos ou +ligações de diretório e copiar toda a árvore de dados congelada, duplicando +dados e esgotando o armazenamento de backup. Configure sempre as +ferramentas de cópia de segurança externas para excluir o diretório +`.snapraid/`. + +No ZFS, os instantâneos são expostos através do diretório virtual padrão +`.zfs/snapshot/` em vez de `.snapraid/`. Por predefinição (`snapdir=hidden`), +o ZFS oculta esse diretório das listagens, de modo que ferramentas externas +de cópia de segurança não o percorrem. Se tiver ativado `snapdir=visible`, +certifique-se de que as suas ferramentas de cópia de segurança excluem o +diretório `.zfs/`. + 10 PADRãO (PATTERN) =================== diff --git a/doc/snapraid-ro.1 b/doc/snapraid-ro.1 index a9036c6f..1d509188 100644 --- a/doc/snapraid-ro.1 +++ b/doc/snapraid-ro.1 @@ -1952,6 +1952,36 @@ cu succes. Paritatea poate conține în continuare blocuri calculate din snapshot\-ul utilizat de sync\-ul întrerupt. Un sync ulterior poate înlocui acel snapshot, făcând versiunile intermediare ale fișierelor indisponibile pentru recuperare. +.SS Copii de rezervă externe cu snapshot\-uri de sistem de fișiere (External backups with filesystem snapshots) +Când snapshot\-urile sistemului de fișiere sunt activate, SnapRAID accesează +sau creează snapshot\-uri la rădăcina fiecărui subvolum, volum sau set de +date (dataset). +.PP +O modalitate simplă de a evita orice interferență cu instrumentele externe +este de a configura discul \`data\` astfel încât să indice spre un subdirector +obișnuit din cadrul subvolumului sau volumului (de exemplu +\`/mnt/subvol/data\` în loc de \`/mnt/subvol\`). SnapRAID creează întotdeauna +directorul de snapshot\-uri la rădăcina subvolumului (de exemplu +\`/mnt/subvol/.snapraid/\`), lăsând subdirectorul de date curat, astfel încât +snapshot\-urile să rămână în exterior și invizibile pentru instrumentele de +backup care operează în interiorul său. +.PP +În caz contrar, dacă directorul \`data\` configurat este chiar rădăcina +subvolumului sau a volumului, SnapRAID creează un director \`.snapraid/\` +direct în interiorul acestuia (pe Btrfs, Bcachefs și NTFS). Deși SnapRAID +exclude automat acest director din scanarea proprie, instrumentele externe +de backup sau sincronizare îl vor întâlni. Dacă nu este exclus, acestea pot +traversa snapshot\-urile sau legăturile de directoare și pot copia întregul +arbore de date înghețat, duplicând datele și epuizând spațiul de stocare al +backup\-ului. Configurați întotdeauna instrumentele externe de backup să +excludă directorul \`.snapraid/\`. +.PP +Pe ZFS, snapshot\-urile sunt expuse prin directorul virtual standard +\`.zfs/snapshot/\` în loc de \`.snapraid/\`. În mod implicit (\`snapdir=hidden\`), +ZFS ascunde acest director din listările de directoare, astfel încât +instrumentele externe de backup nu îl vor parcurge. Dacă ați activat +\`snapdir=visible\`, asigurați\-vă că instrumentele de backup exclud +directorul \`.zfs/\`. .SH MODEL (PATTERN) Modelele oferă o modalitate flexibilă de a filtra fișierele pentru includere sau excludere. Folosind caractere de tip globbing, puteți defini reguli care să diff --git a/doc/snapraid-ro.d b/doc/snapraid-ro.d index 3da8a3a9..e9d89159 100644 --- a/doc/snapraid-ro.d +++ b/doc/snapraid-ro.d @@ -1557,6 +1557,37 @@ Ciclul de viață al snapshot-urilor (Snapshots Lifecycle) acel snapshot, făcând versiunile intermediare ale fișierelor indisponibile pentru recuperare. + Copii de rezervă externe cu snapshot-uri de sistem de fișiere (External backups with filesystem snapshots) + Când snapshot-urile sistemului de fișiere sunt activate, SnapRAID accesează + sau creează snapshot-uri la rădăcina fiecărui subvolum, volum sau set de + date (dataset). + + O modalitate simplă de a evita orice interferență cu instrumentele externe + este de a configura discul `data` astfel încât să indice spre un subdirector + obișnuit din cadrul subvolumului sau volumului (de exemplu + `/mnt/subvol/data` în loc de `/mnt/subvol`). SnapRAID creează întotdeauna + directorul de snapshot-uri la rădăcina subvolumului (de exemplu + `/mnt/subvol/.snapraid/`), lăsând subdirectorul de date curat, astfel încât + snapshot-urile să rămână în exterior și invizibile pentru instrumentele de + backup care operează în interiorul său. + + În caz contrar, dacă directorul `data` configurat este chiar rădăcina + subvolumului sau a volumului, SnapRAID creează un director `.snapraid/` + direct în interiorul acestuia (pe Btrfs, Bcachefs și NTFS). Deși SnapRAID + exclude automat acest director din scanarea proprie, instrumentele externe + de backup sau sincronizare îl vor întâlni. Dacă nu este exclus, acestea pot + traversa snapshot-urile sau legăturile de directoare și pot copia întregul + arbore de date înghețat, duplicând datele și epuizând spațiul de stocare al + backup-ului. Configurați întotdeauna instrumentele externe de backup să + excludă directorul `.snapraid/`. + + Pe ZFS, snapshot-urile sunt expuse prin directorul virtual standard + `.zfs/snapshot/` în loc de `.snapraid/`. În mod implicit (`snapdir=hidden`), + ZFS ascunde acest director din listările de directoare, astfel încât + instrumentele externe de backup nu îl vor parcurge. Dacă ați activat + `snapdir=visible`, asigurați-vă că instrumentele de backup exclud + directorul `.zfs/`. + Model (Pattern) Modelele oferă o modalitate flexibilă de a filtra fișierele pentru includere sau excludere. Folosind caractere de tip globbing, puteți defini reguli care să diff --git a/doc/snapraid-ro.txt b/doc/snapraid-ro.txt index 6a3e3880..c9d9a2ff 100644 --- a/doc/snapraid-ro.txt +++ b/doc/snapraid-ro.txt @@ -1667,6 +1667,39 @@ snapshot-ul utilizat de sync-ul întrerupt. Un sync ulterior poate înlocui acel snapshot, făcând versiunile intermediare ale fișierelor indisponibile pentru recuperare. +9.1 Copii de rezervă externe cu snapshot-uri de sistem de fișiere (External backups with filesystem snapshots) +-------------------------------------------------------------------------------------------------------------- + +Când snapshot-urile sistemului de fișiere sunt activate, SnapRAID accesează +sau creează snapshot-uri la rădăcina fiecărui subvolum, volum sau set de +date (dataset). + +O modalitate simplă de a evita orice interferență cu instrumentele externe +este de a configura discul `data` astfel încât să indice spre un subdirector +obișnuit din cadrul subvolumului sau volumului (de exemplu +`/mnt/subvol/data` în loc de `/mnt/subvol`). SnapRAID creează întotdeauna +directorul de snapshot-uri la rădăcina subvolumului (de exemplu +`/mnt/subvol/.snapraid/`), lăsând subdirectorul de date curat, astfel încât +snapshot-urile să rămână în exterior și invizibile pentru instrumentele de +backup care operează în interiorul său. + +În caz contrar, dacă directorul `data` configurat este chiar rădăcina +subvolumului sau a volumului, SnapRAID creează un director `.snapraid/` +direct în interiorul acestuia (pe Btrfs, Bcachefs și NTFS). Deși SnapRAID +exclude automat acest director din scanarea proprie, instrumentele externe +de backup sau sincronizare îl vor întâlni. Dacă nu este exclus, acestea pot +traversa snapshot-urile sau legăturile de directoare și pot copia întregul +arbore de date înghețat, duplicând datele și epuizând spațiul de stocare al +backup-ului. Configurați întotdeauna instrumentele externe de backup să +excludă directorul `.snapraid/`. + +Pe ZFS, snapshot-urile sunt expuse prin directorul virtual standard +`.zfs/snapshot/` în loc de `.snapraid/`. În mod implicit (`snapdir=hidden`), +ZFS ascunde acest director din listările de directoare, astfel încât +instrumentele externe de backup nu îl vor parcurge. Dacă ați activat +`snapdir=visible`, asigurați-vă că instrumentele de backup exclud +directorul `.zfs/`. + 10 MODEL (PATTERN) ================== diff --git a/doc/snapraid-ru.1 b/doc/snapraid-ru.1 index bcfb9ca0..4b5e257f 100644 --- a/doc/snapraid-ru.1 +++ b/doc/snapraid-ru.1 @@ -1957,6 +1957,34 @@ SnapRAID управляет тремя конкретными снимками, использованного прерванной синхронизацией. Более поздняя синхронизация может заменить этот снимок, сделав промежуточные версии файлов недоступными для восстановления. +.SS Внешнее резервное копирование со снимками файловой системы (External backups with filesystem snapshots) +Когда снимки файловой системы включены, SnapRAID обращается к снимкам или +создает их в корне каждого подтома, тома или набора данных (dataset). +.PP +Простой способ избежать любых конфликтов со сторонними инструментами — +указать в директиве \`data\` обычный подкаталог внутри подтома или тома +(например, \`/mnt/subvol/data\` вместо \`/mnt/subvol\`). SnapRAID всегда +создает каталог снимков в корне подтома (например, +\`/mnt/subvol/.snapraid/\`), оставив подкаталог данных чистым, благодаря +чему снимки останутся снаружи и будут невидимы для программ резервного +копирования, работающих внутри него. +.PP +В противном случае, если настроенный каталог \`data\` сам по себе является +корнем подтома или тома, SnapRAID создает каталог \`.snapraid/\` прямо внутри +него (в Btrfs, Bcachefs и NTFS). Хотя SnapRAID автоматически исключает этот +каталог из собственного сканирования, внешние утилиты резервного +копирования или синхронизации файлов обнаружат его. Если каталог не +исключен, они могут войти в снимки или ссылки на каталоги и скопировать +все замороженное дерево данных, дублируя файлы и исчерпывая место в +хранилище резервных копий. Всегда настраивайте внешние утилиты резервного +копирования на исключение каталога \`.snapraid/\`. +.PP +В ZFS снимки доступны через стандартный виртуальный каталог +\`.zfs/snapshot/\`, а не через \`.snapraid/\`. По умолчанию (\`snapdir=hidden\`) +ZFS скрывает этот каталог из списков директорий, поэтому внешние утилиты +резервного копирования не будут сканировать его. Если вы включили +\`snapdir=visible\`, обязательно настройте утилиты резервного копирования на +исключение каталога \`.zfs/\`. .SH Шаблон (PATTERN) Шаблоны обеспечивают гибкий способ фильтрации файлов для включения или исключения. Используя символы подстановки (globbing), вы можете определить правила, diff --git a/doc/snapraid-ru.d b/doc/snapraid-ru.d index c2097a95..2ce06ad5 100644 --- a/doc/snapraid-ru.d +++ b/doc/snapraid-ru.d @@ -1564,6 +1564,35 @@ Name{number} заменить этот снимок, сделав промежуточные версии файлов недоступными для восстановления. + Внешнее резервное копирование со снимками файловой системы (External backups with filesystem snapshots) + Когда снимки файловой системы включены, SnapRAID обращается к снимкам или + создает их в корне каждого подтома, тома или набора данных (dataset). + + Простой способ избежать любых конфликтов со сторонними инструментами — + указать в директиве `data` обычный подкаталог внутри подтома или тома + (например, `/mnt/subvol/data` вместо `/mnt/subvol`). SnapRAID всегда + создает каталог снимков в корне подтома (например, + `/mnt/subvol/.snapraid/`), оставив подкаталог данных чистым, благодаря + чему снимки останутся снаружи и будут невидимы для программ резервного + копирования, работающих внутри него. + + В противном случае, если настроенный каталог `data` сам по себе является + корнем подтома или тома, SnapRAID создает каталог `.snapraid/` прямо внутри + него (в Btrfs, Bcachefs и NTFS). Хотя SnapRAID автоматически исключает этот + каталог из собственного сканирования, внешние утилиты резервного + копирования или синхронизации файлов обнаружат его. Если каталог не + исключен, они могут войти в снимки или ссылки на каталоги и скопировать + все замороженное дерево данных, дублируя файлы и исчерпывая место в + хранилище резервных копий. Всегда настраивайте внешние утилиты резервного + копирования на исключение каталога `.snapraid/`. + + В ZFS снимки доступны через стандартный виртуальный каталог + `.zfs/snapshot/`, а не через `.snapraid/`. По умолчанию (`snapdir=hidden`) + ZFS скрывает этот каталог из списков директорий, поэтому внешние утилиты + резервного копирования не будут сканировать его. Если вы включили + `snapdir=visible`, обязательно настройте утилиты резервного копирования на + исключение каталога `.zfs/`. + Шаблон (Pattern) Шаблоны обеспечивают гибкий способ фильтрации файлов для включения или исключения. Используя символы подстановки (globbing), вы можете определить правила, diff --git a/doc/snapraid-ru.txt b/doc/snapraid-ru.txt index 7133e41b..e1f6a0e6 100644 --- a/doc/snapraid-ru.txt +++ b/doc/snapraid-ru.txt @@ -1674,6 +1674,37 @@ SnapRAID управляет тремя конкретными снимками, заменить этот снимок, сделав промежуточные версии файлов недоступными для восстановления. +9.1 Внешнее резервное копирование со снимками файловой системы (External backups with filesystem snapshots) +----------------------------------------------------------------------------------------------------------- + +Когда снимки файловой системы включены, SnapRAID обращается к снимкам или +создает их в корне каждого подтома, тома или набора данных (dataset). + +Простой способ избежать любых конфликтов со сторонними инструментами — +указать в директиве `data` обычный подкаталог внутри подтома или тома +(например, `/mnt/subvol/data` вместо `/mnt/subvol`). SnapRAID всегда +создает каталог снимков в корне подтома (например, +`/mnt/subvol/.snapraid/`), оставив подкаталог данных чистым, благодаря +чему снимки останутся снаружи и будут невидимы для программ резервного +копирования, работающих внутри него. + +В противном случае, если настроенный каталог `data` сам по себе является +корнем подтома или тома, SnapRAID создает каталог `.snapraid/` прямо внутри +него (в Btrfs, Bcachefs и NTFS). Хотя SnapRAID автоматически исключает этот +каталог из собственного сканирования, внешние утилиты резервного +копирования или синхронизации файлов обнаружат его. Если каталог не +исключен, они могут войти в снимки или ссылки на каталоги и скопировать +все замороженное дерево данных, дублируя файлы и исчерпывая место в +хранилище резервных копий. Всегда настраивайте внешние утилиты резервного +копирования на исключение каталога `.snapraid/`. + +В ZFS снимки доступны через стандартный виртуальный каталог +`.zfs/snapshot/`, а не через `.snapraid/`. По умолчанию (`snapdir=hidden`) +ZFS скрывает этот каталог из списков директорий, поэтому внешние утилиты +резервного копирования не будут сканировать его. Если вы включили +`snapdir=visible`, обязательно настройте утилиты резервного копирования на +исключение каталога `.zfs/`. + 10 Шаблон (PATTERN) =================== diff --git a/doc/snapraid-sv.1 b/doc/snapraid-sv.1 index 9173ff54..769baa66 100644 --- a/doc/snapraid-sv.1 +++ b/doc/snapraid-sv.1 @@ -1953,6 +1953,34 @@ framgångsrikt. Pariteten kan fortfarande innehålla block som beräknats från ögonblicksbilden som användes av den avbrutna synkroniseringen. En senare synkronisering kan ersätta den ögonblicksbilden, vilket gör mellanliggande filversioner otillgängliga för återställning. +.SS Externa säkerhetskopior med ögonblicksbilder av filsystem (External backups with filesystem snapshots) +När ögonblicksbilder av filsystem är aktiverade ansluter eller skapar +SnapRAID ögonblicksbilder vid roten av varje dataundervolym, volym eller +datauppsättning (dataset). +.PP +Ett enkelt sätt att undvika konflikter med externa verktyg är att peka +din \`data\`\-konfiguration till en vanlig underkatalog i undervolymen eller +volymen (till exempel \`/mnt/subvol/data\` istället för \`/mnt/subvol\`). +SnapRAID skapar alltid katalogen för ögonblicksbilder i undervolymens rot +(till exempel \`/mnt/subvol/.snapraid/\`), vilket håller underkatalogen för +data ren så att ögonblicksbilderna förblir utanför och osynliga för +säkerhetskopieringsverktyg som arbetar i den. +.PP +Annars, om din konfigurerade \`data\`\-katalog i sig är roten på undervolymen +eller volymen, skapar SnapRAID en \`.snapraid/\`\-katalog direkt i den (på +Btrfs, Bcachefs och NTFS). Även om SnapRAID automatiskt utesluter denna +katalog från sin egen skanning, kommer externa säkerhetskopieringsverktyg +eller synkroniseringsprogram att stöta på den. Om den inte utesluts kan de +gå in i ögonblicksbilderna eller kataloglänkarna och kopiera hela det frysta +dataträdet, vilket duplicerar data och kan tömma lagringsutrymmet för +säkerhetskopior. Konfigurera alltid externa säkerhetskopieringsverktyg till +att utesluta katalogen \`.snapraid/\`. +.PP +På ZFS exponeras ögonblicksbilder via den virtuella standardkatalogen +\`.zfs/snapshot/\` istället för \`.snapraid/\`. Som standard (\`snapdir=hidden\`) +döljer ZFS denna katalog från kataloglistor så att externa säkerhetskopieringsverktyg +inte söker igenom den. Om du har aktiverat \`snapdir=visible\`, se till att dina +säkerhetskopieringsverktyg utesluter katalogen \`.zfs/\`. .SH MöNSTER (PATTERN) Mönster ger ett flexibelt sätt att filtrera filer för inkludering eller exkludering. Genom att använda jokertecken kan du definiera regler som diff --git a/doc/snapraid-sv.d b/doc/snapraid-sv.d index cba81745..c49f0796 100644 --- a/doc/snapraid-sv.d +++ b/doc/snapraid-sv.d @@ -1558,6 +1558,35 @@ Livscykel för ögonblicksbilder (Snapshots Lifecycle) synkronisering kan ersätta den ögonblicksbilden, vilket gör mellanliggande filversioner otillgängliga för återställning. + Externa säkerhetskopior med ögonblicksbilder av filsystem (External backups with filesystem snapshots) + När ögonblicksbilder av filsystem är aktiverade ansluter eller skapar + SnapRAID ögonblicksbilder vid roten av varje dataundervolym, volym eller + datauppsättning (dataset). + + Ett enkelt sätt att undvika konflikter med externa verktyg är att peka + din `data`-konfiguration till en vanlig underkatalog i undervolymen eller + volymen (till exempel `/mnt/subvol/data` istället för `/mnt/subvol`). + SnapRAID skapar alltid katalogen för ögonblicksbilder i undervolymens rot + (till exempel `/mnt/subvol/.snapraid/`), vilket håller underkatalogen för + data ren så att ögonblicksbilderna förblir utanför och osynliga för + säkerhetskopieringsverktyg som arbetar i den. + + Annars, om din konfigurerade `data`-katalog i sig är roten på undervolymen + eller volymen, skapar SnapRAID en `.snapraid/`-katalog direkt i den (på + Btrfs, Bcachefs och NTFS). Även om SnapRAID automatiskt utesluter denna + katalog från sin egen skanning, kommer externa säkerhetskopieringsverktyg + eller synkroniseringsprogram att stöta på den. Om den inte utesluts kan de + gå in i ögonblicksbilderna eller kataloglänkarna och kopiera hela det frysta + dataträdet, vilket duplicerar data och kan tömma lagringsutrymmet för + säkerhetskopior. Konfigurera alltid externa säkerhetskopieringsverktyg till + att utesluta katalogen `.snapraid/`. + + På ZFS exponeras ögonblicksbilder via den virtuella standardkatalogen + `.zfs/snapshot/` istället för `.snapraid/`. Som standard (`snapdir=hidden`) + döljer ZFS denna katalog från kataloglistor så att externa säkerhetskopieringsverktyg + inte söker igenom den. Om du har aktiverat `snapdir=visible`, se till att dina + säkerhetskopieringsverktyg utesluter katalogen `.zfs/`. + Mönster (Pattern) Mönster ger ett flexibelt sätt att filtrera filer för inkludering eller exkludering. Genom att använda jokertecken kan du definiera regler som diff --git a/doc/snapraid-sv.txt b/doc/snapraid-sv.txt index beb8e6a1..59bec6b3 100644 --- a/doc/snapraid-sv.txt +++ b/doc/snapraid-sv.txt @@ -1668,6 +1668,37 @@ framgångsrikt. Pariteten kan fortfarande innehålla block som beräknats från synkronisering kan ersätta den ögonblicksbilden, vilket gör mellanliggande filversioner otillgängliga för återställning. +9.1 Externa säkerhetskopior med ögonblicksbilder av filsystem (External backups with filesystem snapshots) +---------------------------------------------------------------------------------------------------------- + +När ögonblicksbilder av filsystem är aktiverade ansluter eller skapar +SnapRAID ögonblicksbilder vid roten av varje dataundervolym, volym eller +datauppsättning (dataset). + +Ett enkelt sätt att undvika konflikter med externa verktyg är att peka +din `data`-konfiguration till en vanlig underkatalog i undervolymen eller +volymen (till exempel `/mnt/subvol/data` istället för `/mnt/subvol`). +SnapRAID skapar alltid katalogen för ögonblicksbilder i undervolymens rot +(till exempel `/mnt/subvol/.snapraid/`), vilket håller underkatalogen för +data ren så att ögonblicksbilderna förblir utanför och osynliga för +säkerhetskopieringsverktyg som arbetar i den. + +Annars, om din konfigurerade `data`-katalog i sig är roten på undervolymen +eller volymen, skapar SnapRAID en `.snapraid/`-katalog direkt i den (på +Btrfs, Bcachefs och NTFS). Även om SnapRAID automatiskt utesluter denna +katalog från sin egen skanning, kommer externa säkerhetskopieringsverktyg +eller synkroniseringsprogram att stöta på den. Om den inte utesluts kan de +gå in i ögonblicksbilderna eller kataloglänkarna och kopiera hela det frysta +dataträdet, vilket duplicerar data och kan tömma lagringsutrymmet för +säkerhetskopior. Konfigurera alltid externa säkerhetskopieringsverktyg till +att utesluta katalogen `.snapraid/`. + +På ZFS exponeras ögonblicksbilder via den virtuella standardkatalogen +`.zfs/snapshot/` istället för `.snapraid/`. Som standard (`snapdir=hidden`) +döljer ZFS denna katalog från kataloglistor så att externa säkerhetskopieringsverktyg +inte söker igenom den. Om du har aktiverat `snapdir=visible`, se till att dina +säkerhetskopieringsverktyg utesluter katalogen `.zfs/`. + 10 MöNSTER (PATTERN) ==================== diff --git a/doc/snapraid-uk.1 b/doc/snapraid-uk.1 index 766ceae0..ba98c152 100644 --- a/doc/snapraid-uk.1 +++ b/doc/snapraid-uk.1 @@ -2126,6 +2126,34 @@ SnapRAID керує трьома конкретними знімками, \`stab використаного перерваною синхронізацією. Пізніша синхронізація може замінити цей знімок, зробивши проміжні версії файлів недоступними для відновлення. +.SS Зовнішнє резервне копіювання зі знімками файлової системи (External backups with filesystem snapshots) +Коли знімки файлової системи ввімкнено, SnapRAID звертається до знімків або +створює їх у корені кожного підтому даних, тому або набору даних (dataset). +.PP +Простий спосіб уникнути будь\-яких конфліктів зі сторонніми інструментами — +вказати в налаштуванні \`data\` звичайний підкаталог усередині підтому або +тому (наприклад, \`/mnt/subvol/data\` замість \`/mnt/subvol\`). SnapRAID завжди +створює каталог знімків у корені підтому (наприклад, +\`/mnt/subvol/.snapraid/\`), залишивши підкаталог даних чистим, завдяки чому +знімки залишаться ззовні та будуть невидимими для засобів резервного +копіювання, що працюють у ньому. +.PP +В іншому випадку, якщо налаштований каталог \`data\` сам по собі є коренем +підтому або тому, SnapRAID створює каталог \`.snapraid/\` безпосередньо +всередині нього (у Btrfs, Bcachefs та NTFS). Хоча SnapRAID автоматично +виключає цей каталог із власного сканування, зовнішні інструменти +резервного копіювання або синхронізації файлів виявлять його. Якщо каталог +не виключено, вони можуть увійти у знімки або посилання на каталоги та +скопіювати все заморожене дерево даних, дублюючи файли й вичерпуючи +дисковий простір резервного сховища. Завжди налаштовуйте зовнішні засоби +резервного копіювання на виключення каталогу \`.snapraid/\`. +.PP +У ZFS знімки надаються через стандартний віртуальний каталог +\`.zfs/snapshot/\` замість \`.snapraid/\`. За замовчуванням (\`snapdir=hidden\`) +ZFS приховує цей каталог зі списків каталогів, тому зовнішні інструменти +резервного копіювання не скануватимуть його. Якщо ви ввімкнули +\`snapdir=visible\`, обов\'язково переконайтеся, що ваші інструменти +резервного копіювання виключають каталог \`.zfs/\`. .SH Шаблон (PATTERN) Шаблони забезпечують гнучкий спосіб фільтрації файлів для включення або виключення. Використовуючи символи підстановки (globbing), ви можете визначити правила, які diff --git a/doc/snapraid-uk.d b/doc/snapraid-uk.d index 9e6df76c..e06dcff0 100644 --- a/doc/snapraid-uk.d +++ b/doc/snapraid-uk.d @@ -1722,6 +1722,35 @@ Name{number} замінити цей знімок, зробивши проміжні версії файлів недоступними для відновлення. + Зовнішнє резервне копіювання зі знімками файлової системи (External backups with filesystem snapshots) + Коли знімки файлової системи ввімкнено, SnapRAID звертається до знімків або + створює їх у корені кожного підтому даних, тому або набору даних (dataset). + + Простий спосіб уникнути будь-яких конфліктів зі сторонніми інструментами — + вказати в налаштуванні `data` звичайний підкаталог усередині підтому або + тому (наприклад, `/mnt/subvol/data` замість `/mnt/subvol`). SnapRAID завжди + створює каталог знімків у корені підтому (наприклад, + `/mnt/subvol/.snapraid/`), залишивши підкаталог даних чистим, завдяки чому + знімки залишаться ззовні та будуть невидимими для засобів резервного + копіювання, що працюють у ньому. + + В іншому випадку, якщо налаштований каталог `data` сам по собі є коренем + підтому або тому, SnapRAID створює каталог `.snapraid/` безпосередньо + всередині нього (у Btrfs, Bcachefs та NTFS). Хоча SnapRAID автоматично + виключає цей каталог із власного сканування, зовнішні інструменти + резервного копіювання або синхронізації файлів виявлять його. Якщо каталог + не виключено, вони можуть увійти у знімки або посилання на каталоги та + скопіювати все заморожене дерево даних, дублюючи файли й вичерпуючи + дисковий простір резервного сховища. Завжди налаштовуйте зовнішні засоби + резервного копіювання на виключення каталогу `.snapraid/`. + + У ZFS знімки надаються через стандартний віртуальний каталог + `.zfs/snapshot/` замість `.snapraid/`. За замовчуванням (`snapdir=hidden`) + ZFS приховує цей каталог зі списків каталогів, тому зовнішні інструменти + резервного копіювання не скануватимуть його. Якщо ви ввімкнули + `snapdir=visible`, обов'язково переконайтеся, що ваші інструменти + резервного копіювання виключають каталог `.zfs/`. + Шаблон (Pattern) Шаблони забезпечують гнучкий спосіб фільтрації файлів для включення або виключення. Використовуючи символи підстановки (globbing), ви можете визначити правила, які diff --git a/doc/snapraid-uk.txt b/doc/snapraid-uk.txt index 95d034d8..1dcdc967 100644 --- a/doc/snapraid-uk.txt +++ b/doc/snapraid-uk.txt @@ -1832,6 +1832,37 @@ SnapRAID керує трьома конкретними знімками, `stabl замінити цей знімок, зробивши проміжні версії файлів недоступними для відновлення. +9.1 Зовнішнє резервне копіювання зі знімками файлової системи (External backups with filesystem snapshots) +---------------------------------------------------------------------------------------------------------- + +Коли знімки файлової системи ввімкнено, SnapRAID звертається до знімків або +створює їх у корені кожного підтому даних, тому або набору даних (dataset). + +Простий спосіб уникнути будь-яких конфліктів зі сторонніми інструментами — +вказати в налаштуванні `data` звичайний підкаталог усередині підтому або +тому (наприклад, `/mnt/subvol/data` замість `/mnt/subvol`). SnapRAID завжди +створює каталог знімків у корені підтому (наприклад, +`/mnt/subvol/.snapraid/`), залишивши підкаталог даних чистим, завдяки чому +знімки залишаться ззовні та будуть невидимими для засобів резервного +копіювання, що працюють у ньому. + +В іншому випадку, якщо налаштований каталог `data` сам по собі є коренем +підтому або тому, SnapRAID створює каталог `.snapraid/` безпосередньо +всередині нього (у Btrfs, Bcachefs та NTFS). Хоча SnapRAID автоматично +виключає цей каталог із власного сканування, зовнішні інструменти +резервного копіювання або синхронізації файлів виявлять його. Якщо каталог +не виключено, вони можуть увійти у знімки або посилання на каталоги та +скопіювати все заморожене дерево даних, дублюючи файли й вичерпуючи +дисковий простір резервного сховища. Завжди налаштовуйте зовнішні засоби +резервного копіювання на виключення каталогу `.snapraid/`. + +У ZFS знімки надаються через стандартний віртуальний каталог +`.zfs/snapshot/` замість `.snapraid/`. За замовчуванням (`snapdir=hidden`) +ZFS приховує цей каталог зі списків каталогів, тому зовнішні інструменти +резервного копіювання не скануватимуть його. Якщо ви ввімкнули +`snapdir=visible`, обов'язково переконайтеся, що ваші інструменти +резервного копіювання виключають каталог `.zfs/`. + 10 Шаблон (PATTERN) =================== diff --git a/doc/snapraid-zh.1 b/doc/snapraid-zh.1 index 1359424b..c39b06ed 100644 --- a/doc/snapraid-zh.1 +++ b/doc/snapraid-zh.1 @@ -1755,6 +1755,25 @@ SnapRAID 在每个数据子卷根目录下的隐藏目录中管理三个特定 但是,如果 \`sync\` 中断,请避免在后续 \`sync\` 成功完成之前修改、删除、 替换或移动文件。校验可能仍包含根据中断的同步所使用的快照计算的块。 以后的同步可能会替换该快照,从而导致中间文件版本无法用于恢复。 +.SS 使用文件系统快照进行外部备份 (External backups with filesystem snapshots) +当启用文件系统快照时,SnapRAID 会在每个数据子卷、卷或数据集(dataset) +的根目录下访问或创建快照。 +.PP +避免与外部工具发生冲突的一种简单方法,是将 \`data\` 配置指向子卷或卷内的 +常规子目录(例如 \`/mnt/subvol/data\` 而不是 \`/mnt/subvol\`)。SnapRAID +始终会在子卷根目录下创建快照目录(例如 \`/mnt/subvol/.snapraid/\`),从而保持 +数据子目录整洁,使快照保留在外部且对在该目录内运行的备份工具不可见。 +.PP +否则,如果配置的 \`data\` 目录本身就是子卷或卷的根目录,SnapRAID 会直接在其中 +创建 \`.snapraid/\` 目录(在 Btrfs、Bcachefs 和 NTFS 上)。虽然 SnapRAID +会在其自身的扫描中自动排除此目录,但外部备份工具或文件同步工具会发现它。如果未 +将其排除,它们可能会遍历快照或目录链接并复制整个冻结的数据树,从而导致数据 +重复并耗尽备份存储空间。请务必将外部备份工具配置为排除 \`.snapraid/\` 目录。 +.PP +在 ZFS 上,快照是通过标准虚拟目录 \`.zfs/snapshot/\` 而不是 \`.snapraid/\` +公开的。默认情况下(\`snapdir=hidden\`),ZFS 会在目录列表中隐藏此目录,因此 +外部备份工具不会遍历它。如果您启用了 \`snapdir=visible\`,请确保您的备份工具 +排除 \`.zfs/\` 目录。 .SH 模式 (PATTERN) 模式提供了一种灵活的方法来过滤要包含或排除的文件。通过使用通配符(globbing), 您可以定义匹配特定文件名或整个目录结构的规则,而无需手动列出每个路径。 diff --git a/doc/snapraid-zh.d b/doc/snapraid-zh.d index 324c5511..f2b0836e 100644 --- a/doc/snapraid-zh.d +++ b/doc/snapraid-zh.d @@ -1360,6 +1360,26 @@ Name{number} 替换或移动文件。校验可能仍包含根据中断的同步所使用的快照计算的块。 以后的同步可能会替换该快照,从而导致中间文件版本无法用于恢复。 + 使用文件系统快照进行外部备份 (External backups with filesystem snapshots) + 当启用文件系统快照时,SnapRAID 会在每个数据子卷、卷或数据集(dataset) + 的根目录下访问或创建快照。 + + 避免与外部工具发生冲突的一种简单方法,是将 `data` 配置指向子卷或卷内的 + 常规子目录(例如 `/mnt/subvol/data` 而不是 `/mnt/subvol`)。SnapRAID + 始终会在子卷根目录下创建快照目录(例如 `/mnt/subvol/.snapraid/`),从而保持 + 数据子目录整洁,使快照保留在外部且对在该目录内运行的备份工具不可见。 + + 否则,如果配置的 `data` 目录本身就是子卷或卷的根目录,SnapRAID 会直接在其中 + 创建 `.snapraid/` 目录(在 Btrfs、Bcachefs 和 NTFS 上)。虽然 SnapRAID + 会在其自身的扫描中自动排除此目录,但外部备份工具或文件同步工具会发现它。如果未 + 将其排除,它们可能会遍历快照或目录链接并复制整个冻结的数据树,从而导致数据 + 重复并耗尽备份存储空间。请务必将外部备份工具配置为排除 `.snapraid/` 目录。 + + 在 ZFS 上,快照是通过标准虚拟目录 `.zfs/snapshot/` 而不是 `.snapraid/` + 公开的。默认情况下(`snapdir=hidden`),ZFS 会在目录列表中隐藏此目录,因此 + 外部备份工具不会遍历它。如果您启用了 `snapdir=visible`,请确保您的备份工具 + 排除 `.zfs/` 目录。 + 模式 (Pattern) 模式提供了一种灵活的方法来过滤要包含或排除的文件。通过使用通配符(globbing), 您可以定义匹配特定文件名或整个目录结构的规则,而无需手动列出每个路径。 diff --git a/doc/snapraid-zh.txt b/doc/snapraid-zh.txt index 76a04230..4791ecd4 100644 --- a/doc/snapraid-zh.txt +++ b/doc/snapraid-zh.txt @@ -1470,6 +1470,28 @@ SnapRAID 在每个数据子卷根目录下的隐藏目录中管理三个特定 替换或移动文件。校验可能仍包含根据中断的同步所使用的快照计算的块。 以后的同步可能会替换该快照,从而导致中间文件版本无法用于恢复。 +9.1 使用文件系统快照进行外部备份 (External backups with filesystem snapshots) +--------------------------------------------------------------- + +当启用文件系统快照时,SnapRAID 会在每个数据子卷、卷或数据集(dataset) +的根目录下访问或创建快照。 + +避免与外部工具发生冲突的一种简单方法,是将 `data` 配置指向子卷或卷内的 +常规子目录(例如 `/mnt/subvol/data` 而不是 `/mnt/subvol`)。SnapRAID +始终会在子卷根目录下创建快照目录(例如 `/mnt/subvol/.snapraid/`),从而保持 +数据子目录整洁,使快照保留在外部且对在该目录内运行的备份工具不可见。 + +否则,如果配置的 `data` 目录本身就是子卷或卷的根目录,SnapRAID 会直接在其中 +创建 `.snapraid/` 目录(在 Btrfs、Bcachefs 和 NTFS 上)。虽然 SnapRAID +会在其自身的扫描中自动排除此目录,但外部备份工具或文件同步工具会发现它。如果未 +将其排除,它们可能会遍历快照或目录链接并复制整个冻结的数据树,从而导致数据 +重复并耗尽备份存储空间。请务必将外部备份工具配置为排除 `.snapraid/` 目录。 + +在 ZFS 上,快照是通过标准虚拟目录 `.zfs/snapshot/` 而不是 `.snapraid/` +公开的。默认情况下(`snapdir=hidden`),ZFS 会在目录列表中隐藏此目录,因此 +外部备份工具不会遍历它。如果您启用了 `snapdir=visible`,请确保您的备份工具 +排除 `.zfs/` 目录。 + 10 模式 (PATTERN) =============== diff --git a/doc/snapraid.1 b/doc/snapraid.1 index e361f5f9..6bd30dce 100644 --- a/doc/snapraid.1 +++ b/doc/snapraid.1 @@ -1962,6 +1962,33 @@ successfully. Parity may still contain blocks computed from the snapshot used by the interrupted sync. A later sync may replace that snapshot, making intermediate file versions unavailable for recovery. +.SS External backups with filesystem snapshots +When filesystem snapshots are enabled, SnapRAID accesses or creates +snapshots at the root of each data subvolume, volume, or dataset. +.PP +A simple way to avoid any interference with external tools is to point +your \`data\` configuration to a regular subdirectory within the subvolume +or volume (for example, \`/mnt/subvol/data\` instead of \`/mnt/subvol\`). +SnapRAID always creates the snapshot directory at the subvolume root +(for example, \`/mnt/subvol/.snapraid/\`), leaving the data subdirectory +clean so that snapshots remain outside and invisible to backup tools +operating inside it. +.PP +Otherwise, if your configured \`data\` directory is itself the subvolume +or volume root, SnapRAID creates the \`.snapraid/\` directory directly +inside it (on Btrfs, Bcachefs, and NTFS). While SnapRAID automatically +excludes this directory from its own scanning, external backup tools +or file synchronizers will encounter it. If not excluded, they may +traverse into the snapshots or directory links and copy the entire +frozen data tree, duplicating data and potentially exhausting backup +storage. Always configure external backup tools to exclude the +\`.snapraid/\` directory. +.PP +On ZFS, snapshots are exposed through the standard \`.zfs/snapshot/\` +virtual directory rather than \`.snapraid/\`. By default (\`snapdir=hidden\`), +ZFS hides this directory from directory listings, so external backup +tools will not traverse it. If you have enabled \`snapdir=visible\`, +ensure your backup tools exclude the \`.zfs/\` directory. .SH PATTERN Patterns are used to select a subset of files to exclude or include in the process. Globbing characters can be used to match files and paths diff --git a/doc/snapraid.d b/doc/snapraid.d index 920ac71d..2682257e 100644 --- a/doc/snapraid.d +++ b/doc/snapraid.d @@ -1567,6 +1567,34 @@ Snapshots Lifecycle that snapshot, making intermediate file versions unavailable for recovery. + External backups with filesystem snapshots + When filesystem snapshots are enabled, SnapRAID accesses or creates + snapshots at the root of each data subvolume, volume, or dataset. + + A simple way to avoid any interference with external tools is to point + your `data` configuration to a regular subdirectory within the subvolume + or volume (for example, `/mnt/subvol/data` instead of `/mnt/subvol`). + SnapRAID always creates the snapshot directory at the subvolume root + (for example, `/mnt/subvol/.snapraid/`), leaving the data subdirectory + clean so that snapshots remain outside and invisible to backup tools + operating inside it. + + Otherwise, if your configured `data` directory is itself the subvolume + or volume root, SnapRAID creates the `.snapraid/` directory directly + inside it (on Btrfs, Bcachefs, and NTFS). While SnapRAID automatically + excludes this directory from its own scanning, external backup tools + or file synchronizers will encounter it. If not excluded, they may + traverse into the snapshots or directory links and copy the entire + frozen data tree, duplicating data and potentially exhausting backup + storage. Always configure external backup tools to exclude the + `.snapraid/` directory. + + On ZFS, snapshots are exposed through the standard `.zfs/snapshot/` + virtual directory rather than `.snapraid/`. By default (`snapdir=hidden`), + ZFS hides this directory from directory listings, so external backup + tools will not traverse it. If you have enabled `snapdir=visible`, + ensure your backup tools exclude the `.zfs/` directory. + Pattern Patterns are used to select a subset of files to exclude or include in the process. Globbing characters can be used to match files and paths diff --git a/doc/snapraid.txt b/doc/snapraid.txt index 4f425fd3..6833e6f0 100644 --- a/doc/snapraid.txt +++ b/doc/snapraid.txt @@ -1677,6 +1677,36 @@ snapshot used by the interrupted sync. A later sync may replace that snapshot, making intermediate file versions unavailable for recovery. +9.1 External backups with filesystem snapshots +---------------------------------------------- + +When filesystem snapshots are enabled, SnapRAID accesses or creates +snapshots at the root of each data subvolume, volume, or dataset. + +A simple way to avoid any interference with external tools is to point +your `data` configuration to a regular subdirectory within the subvolume +or volume (for example, `/mnt/subvol/data` instead of `/mnt/subvol`). +SnapRAID always creates the snapshot directory at the subvolume root +(for example, `/mnt/subvol/.snapraid/`), leaving the data subdirectory +clean so that snapshots remain outside and invisible to backup tools +operating inside it. + +Otherwise, if your configured `data` directory is itself the subvolume +or volume root, SnapRAID creates the `.snapraid/` directory directly +inside it (on Btrfs, Bcachefs, and NTFS). While SnapRAID automatically +excludes this directory from its own scanning, external backup tools +or file synchronizers will encounter it. If not excluded, they may +traverse into the snapshots or directory links and copy the entire +frozen data tree, duplicating data and potentially exhausting backup +storage. Always configure external backup tools to exclude the +`.snapraid/` directory. + +On ZFS, snapshots are exposed through the standard `.zfs/snapshot/` +virtual directory rather than `.snapraid/`. By default (`snapdir=hidden`), +ZFS hides this directory from directory listings, so external backup +tools will not traverse it. If you have enabled `snapdir=visible`, +ensure your backup tools exclude the `.zfs/` directory. + 10 PATTERN ========== From a7a85a259c1515cbf557f86d82bd65706ac0f9cd Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 2 Oct 2026 10:13:35 +0200 Subject: [PATCH 12/51] Add os_tick_us() and os_tick_ns() monotonic high-resolution timers --- configure.ac | 3 +-- os/mingw.c | 27 +++++++++++++++++++++++++++ os/os.h | 12 ++++++++++++ os/portable.h | 5 ----- os/unix.c | 27 ++++++++++++++++++--------- 5 files changed, 58 insertions(+), 16 deletions(-) diff --git a/configure.ac b/configure.ac index dbc75081..b80ad099 100644 --- a/configure.ac +++ b/configure.ac @@ -71,7 +71,7 @@ AC_HEADER_ASSERT AC_HEADER_SYS_WAIT AC_CHECK_HEADERS([dirent.h stdint.h inttypes.h unistd.h math.h execinfo.h strings.h getopt.h syslog.h grp.h pwd.h io.h alloca.h]) AC_CHECK_HEADERS([sys/file.h sys/sysctl.h sys/ioctl.h sys/time.h sys/types.h sys/mkdev.h sys/sysmacros.h sys/stat.h sys/prctl.h sys/sysinfo.h sys/utsname.h]) -AC_CHECK_HEADERS([linux/fs.h linux/btrfs.h linux/fiemap.h mach/mach_time.h]) +AC_CHECK_HEADERS([linux/fs.h linux/btrfs.h linux/fiemap.h]) # Check for the close_range(...,CLOSE_RANGE_CLOEXEC) AC_CHECK_HEADERS([linux/close_range.h]) @@ -126,7 +126,6 @@ AC_CHECK_FUNCS([fsync posix_fadvise sync_file_range syncfs]) AC_CHECK_FUNCS([getc_unlocked ferror_unlocked]) AC_CHECK_FUNCS([futimes futimens futimesat localtime_r lutimes utimensat]) AC_CHECK_FUNCS([fstatat flock renameat statx]) -AC_CHECK_FUNCS([mach_absolute_time]) AC_CHECK_FUNCS([backtrace backtrace_symbols]) AC_SEARCH_LIBS([clock_gettime], [rt]) AC_CHECK_FUNCS([clock_gettime]) diff --git a/os/mingw.c b/os/mingw.c index 3341e784..e457ea30 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -64,6 +64,7 @@ static windows_key_t last_error; */ static windows_mutex_t tick_lock; static uint64_t tick_last; +static uint64_t tick_frequency; /** * Mutex for process reference publication, termination, and unpublication. @@ -4498,6 +4499,24 @@ uint64_t os_tick_ms(void) return GetTickCount64(); } +uint64_t os_tick_us(void) +{ + uint64_t ticks = os_tick(); + uint64_t sec = ticks / tick_frequency; + uint64_t rem = ticks % tick_frequency; + + return sec * 1000000ULL + (rem * 1000000ULL) / tick_frequency; +} + +uint64_t os_tick_ns(void) +{ + uint64_t ticks = os_tick(); + uint64_t sec = ticks / tick_frequency; + uint64_t rem = ticks % tick_frequency; + + return sec * 1000000000ULL + (rem * 1000000000ULL) / tick_frequency; +} + int os_usleep(uint64_t usec) { while (usec > 0) { @@ -4609,6 +4628,13 @@ void os_init(unsigned opt) os_exit(); } + LARGE_INTEGER freq; + if (!QueryPerformanceFrequency(&freq) || freq.QuadPart <= 0) { + os_syslog(OS_LVL_CRITICAL, "error calling QueryPerformanceFrequency()"); + os_exit(); + } + tick_frequency = freq.QuadPart; + if (windows_mutex_init(&exec_mutex, 0) != 0) { os_syslog(OS_LVL_CRITICAL, "error calling windows_mutex_init()"); os_exit(); @@ -4684,6 +4710,7 @@ void os_done(void) windows_mutex_destroy(&exec_mutex); windows_mutex_destroy(&tick_lock); + tick_frequency = 0; /* restore the normal execution level */ SetThreadExecutionState(WIN32_ES_CONTINUOUS); diff --git a/os/os.h b/os/os.h index a0544181..8befff56 100644 --- a/os/os.h +++ b/os/os.h @@ -397,6 +397,18 @@ uint64_t os_tick(void); */ uint64_t os_tick_ms(void); +/** + * Get the os_tick counter value in microsecond. + * \return Monotonic clock value in microseconds. + */ +uint64_t os_tick_us(void); + +/** + * Get the os_tick counter value in nanosecond. + * \return Monotonic clock value in nanoseconds. + */ +uint64_t os_tick_ns(void); + /** * Get the os_tick counter value in seconds. * \return Monotonic clock value in seconds. diff --git a/os/portable.h b/os/portable.h index 1dcd0963..6b86dc78 100644 --- a/os/portable.h +++ b/os/portable.h @@ -132,7 +132,6 @@ #include #endif - /** * Includes some standard headers. */ @@ -174,10 +173,6 @@ #endif #endif -#if HAVE_MACH_MACH_TIME_H -#include -#endif - #if HAVE_DIRENT_H #include #endif diff --git a/os/unix.c b/os/unix.c index bc8e9ae0..41215329 100644 --- a/os/unix.c +++ b/os/unix.c @@ -2217,17 +2217,18 @@ void os_privileges_release(void) uint64_t os_tick(void) { -#if HAVE_MACH_ABSOLUTE_TIME - /* for Mac OS X */ - return mach_absolute_time(); -#elif HAVE_CLOCK_GETTIME && (defined(CLOCK_MONOTONIC) || defined(CLOCK_MONOTONIC_RAW)) - /* for Linux */ +#if HAVE_CLOCK_GETTIME && (defined(CLOCK_UPTIME_RAW) || defined(CLOCK_MONOTONIC_RAW) || defined(CLOCK_MONOTONIC)) struct timespec tv; /* nanosecond precision with clock_gettime() */ -#if defined(CLOCK_MONOTONIC_RAW) +#if defined(CLOCK_UPTIME_RAW) + /* for macOS (without sleep and without NTP slewing) */ + if (clock_gettime(CLOCK_UPTIME_RAW, &tv) != 0) { +#elif defined(CLOCK_MONOTONIC_RAW) + /* for Linux (without sleep and without NTP slewing) */ if (clock_gettime(CLOCK_MONOTONIC_RAW, &tv) != 0) { #else + /* for other POSIX */ if (clock_gettime(CLOCK_MONOTONIC, &tv) != 0) { #endif /* LCOV_EXCL_START */ @@ -2247,7 +2248,7 @@ uint64_t os_tick(void) /* LCOV_EXCL_STOP */ } - return tv.tv_sec * 1000000ULL + tv.tv_usec; + return tv.tv_sec * 1000000000ULL + tv.tv_usec * 1000ULL; #endif } @@ -2276,6 +2277,16 @@ uint64_t os_tick_ms(void) #endif } +uint64_t os_tick_us(void) +{ + return os_tick() / 1000; +} + +uint64_t os_tick_ns(void) +{ + return os_tick(); +} + uint64_t os_tick_sec(void) { #if HAVE_CLOCK_GETTIME && defined(CLOCK_MONOTONIC) @@ -2327,8 +2338,6 @@ int os_usleep(uint64_t usec) return 0; } - - int os_randomize(void* ptr, size_t size) { int f; From dcb3c7db37fc4c5dd88a60fdce6e5ffcc261e0e2 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 2 Oct 2026 10:43:11 +0200 Subject: [PATCH 13/51] Add CONFIG_NEON/32 defines --- os/portable.h | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/os/portable.h b/os/portable.h index 6b86dc78..4ed09558 100644 --- a/os/portable.h +++ b/os/portable.h @@ -84,11 +84,22 @@ #define CONFIG_X86_64 1 #endif -#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) +#if defined(__aarch64__) +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define CONFIG_NEON 1 +#endif +#if defined(__ARM_FEATURE_CRC32) #define CONFIG_ARM_CRC 1 #endif #endif +#if defined(__arm__) +#if defined(__ARM_NEON) || defined(__ARM_NEON__) +#define CONFIG_NEON32 1 +#endif +#endif +#endif + /** * Includes some platform specific headers. */ From 0ad0913ef2bbe3f0097b30311c6c1f1e4bbb09e9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 2 Oct 2026 10:45:30 +0200 Subject: [PATCH 14/51] Use os_tick_ns() monotonic timer and add compiler memory barriers --- cmdline/speed.c | 35 +++++++++++++++-------------------- raid/test/speedtest.c | 18 +++++++++--------- 2 files changed, 24 insertions(+), 29 deletions(-) diff --git a/cmdline/speed.c b/cmdline/speed.c index c690986e..7514ed8a 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -11,18 +11,11 @@ #include "raid/memory.h" #include "state.h" -/** - * Differential us of two timeval. - */ -static int64_t diffgettimeofday(struct timeval* start, struct timeval* stop) -{ - int64_t d; - - d = 1000000LL * (stop->tv_sec - start->tv_sec); - d += stop->tv_usec - start->tv_usec; - - return d; -} +#if HAVE_ASSEMBLY +#define SPEED_BARRIER() __asm__ __volatile__ ("" : : : "memory") +#else +#define SPEED_BARRIER() do { } while (0) +#endif #define SPEED_START \ { \ @@ -30,15 +23,17 @@ static int64_t diffgettimeofday(struct timeval* start, struct timeval* stop) int64_t elapsed_ = 0; \ int64_t best_dt_ = INT64_MAX; \ do { \ - struct timeval start; \ - struct timeval stop; \ - gettimeofday(&start, 0); \ + SPEED_BARRIER(); \ + uint64_t start_ = os_tick_ns(); \ + SPEED_BARRIER(); \ for (int i_ = 0; i_ < speed_delta_; ++i_) #define SPEED_STOP \ - gettimeofday(&stop, 0); \ - dt = diffgettimeofday(&start, &stop); \ - if (dt < 1000 && speed_delta_ <= INT_MAX / 2) { \ + SPEED_BARRIER(); \ + uint64_t stop_ = os_tick_ns(); \ + SPEED_BARRIER(); \ + dt = stop_ > start_ ? stop_ - start_ : 0; \ + if (dt < 1000000 && speed_delta_ <= INT_MAX / 2) { \ speed_delta_ *= 2; \ elapsed_ = 0; \ best_dt_ = INT64_MAX; \ @@ -47,8 +42,8 @@ static int64_t diffgettimeofday(struct timeval* start, struct timeval* stop) elapsed_ += dt; \ if (dt < best_dt_) \ best_dt_ = dt; \ - } while (elapsed_ < period * 1000LL); \ - ds = size * (int64_t)speed_delta_ * nd; \ + } while (elapsed_ < period * 1000000LL); \ + ds = size * (int64_t)speed_delta_ * nd * 1000LL; \ dt = best_dt_; \ } diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 0d40894e..069de5c0 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -36,19 +36,19 @@ static int64_t diffgettimeofday(struct timeval *start, struct timeval *stop) d = 1000000LL * (stop->tv_sec - start->tv_sec); d += stop->tv_usec - start->tv_usec; - return d; + return d > 0 ? d : 0; } #define SPEED_START \ { \ - int speed_delta_ = delta; \ - int64_t elapsed_ = 0; \ - int64_t best_dt_ = INT64_MAX; \ - do { \ - struct timeval start; \ - struct timeval stop; \ - gettimeofday(&start, 0); \ - for (int i_ = 0; i_ < speed_delta_; ++i_) + int speed_delta_ = delta; \ + int64_t elapsed_ = 0; \ + int64_t best_dt_ = INT64_MAX; \ + do { \ + struct timeval start; \ + struct timeval stop; \ + gettimeofday(&start, 0); \ + for (int i_ = 0; i_ < speed_delta_; ++i_) #define SPEED_STOP \ gettimeofday(&stop, 0); \ From 91761768a08fd6c4d5cfa2ddb3c2e70d8abf367d Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 20:16:20 +0200 Subject: [PATCH 15/51] Document rare occurrence of missing info branch in content loading --- cmdline/state.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmdline/state.c b/cmdline/state.c index 781a77c4..1a59a9e3 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2826,6 +2826,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S info_set_run(&state->infoarr, v_pos, v_count, info); v_pos += v_count; } else { + /* + * This branch is entered only when a parity position below blockmax + * has no info, which means that all data disks simultaneously have a + * hole (no file block) at this exact parity position. + * Since blockmax is the highest allocated block across all disks and + * file allocation on disks is typically contiguous from block 0, such + * global parity holes are extremely rare in practice. Even though the + * block-by-block fs_info_is_required() validation is inefficient, it is + * practically never executed, or only for very short gaps. + */ while (v_count) { /* ensure that an info is present only for used positions */ if (fs_info_is_required(state, v_pos)) { From ec71f519b081fe68298dfe097e00378658f5e21a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 5 Oct 2026 11:32:55 +0200 Subject: [PATCH 16/51] Align critical loops to 64 bytes using __align_loops attribute --- cmdline/elem.c | 2 ++ cmdline/elem.h | 1 + cmdline/state.c | 2 ++ cmdline/util.c | 2 ++ cmdline/util.h | 2 ++ os/portable.h | 16 ++++++++++++++++ 6 files changed, 25 insertions(+) diff --git a/cmdline/elem.c b/cmdline/elem.c index 24662185..5d9f1108 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -758,6 +758,7 @@ void extent_free(void* void_extent) free(extent); } +__aligned(32) int extent_parity_compare(const void* void_a, const void* void_b) { const struct snapraid_extent* arg_a = void_a; @@ -771,6 +772,7 @@ int extent_parity_compare(const void* void_a, const void* void_b) return 0; } +__aligned(32) int extent_file_compare(const void* void_a, const void* void_b) { const struct snapraid_extent* arg_a = void_a; diff --git a/cmdline/elem.h b/cmdline/elem.h index faba91b2..81dd62a7 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -1410,6 +1410,7 @@ static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos) * Grow the array once and process each internal block directly, avoiding the * per-position grow and lookup performed by info_set(). */ +__aligned_loops(64) static inline void info_set_run(tommy_arrayblkof* array, block_off_t pos, block_off_t count, snapraid_info info) { tommy_arrayblkof_grow(array, pos + count); diff --git a/cmdline/state.c b/cmdline/state.c index 1a59a9e3..a21e96b4 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2294,6 +2294,7 @@ static void decoding_error(const char* path, STREAM* f) * pointer directly over cached hashes therefore preserves the same integrity * check performed by sread(). */ +__aligned_loops(64) static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count, unsigned state, int discard_hash) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); @@ -3902,6 +3903,7 @@ struct state_write_context { * copied with a single memcpy(). Keep the stream pointer local while filling * each buffered group to avoid the per-block stream call and field updates. */ +__aligned_loops(64) static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); diff --git a/cmdline/util.c b/cmdline/util.c index e1f7ffe3..c6c2e1c4 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -453,6 +453,7 @@ uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, size_t size) } #if CONFIG_X86 +__aligned_loops(64) uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) { crc ^= CRC_IV; @@ -466,6 +467,7 @@ uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) #endif #if CONFIG_ARM_CRC +__aligned_loops(64) uint32_t crc32c_arm64(uint32_t crc, const unsigned char* ptr, size_t size) { crc ^= CRC_IV; diff --git a/cmdline/util.h b/cmdline/util.h index a197f01f..c3f282a7 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -148,6 +148,7 @@ static __always_inline uint32_t crc32c_x86_64_skip(uint32_t crc, const uint32_t ^ table[7][crc >> 28]; } +__aligned_loops(64) static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned char* ptr, size_t lane_size, const uint32_t table[8][16]) { const unsigned char* ptr0 = ptr; @@ -180,6 +181,7 @@ static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned } #endif +__aligned_loops(64) static __always_inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, size_t size) { #ifdef CONFIG_X86_64 diff --git a/os/portable.h b/os/portable.h index 4ed09558..6336600e 100644 --- a/os/portable.h +++ b/os/portable.h @@ -70,6 +70,22 @@ #define __noreturn __attribute__((noreturn)) #endif +#ifndef __aligned +#if defined(__GNUC__) || defined(__clang__) +#define __aligned(n) __attribute__((aligned(n))) +#else +#define __aligned(n) +#endif +#endif + +#ifndef __aligned_loops +#if defined(__GNUC__) && !defined(__clang__) +#define __aligned_loops(n) __attribute__((optimize("align-loops=" #n))) +#else +#define __aligned_loops(n) +#endif +#endif + /** * Architecture for inline assembly. */ From 94c7d3ef9a51f37b4cd4d1169f7b96e05ab91eba Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 4 Oct 2026 22:41:57 +0200 Subject: [PATCH 17/51] Avoid memory preinitialization in malloc_nofail and use calloc directly in calloc_nofail --- cmdline/memory.c | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/cmdline/memory.c b/cmdline/memory.c index 3402287b..3354e2d3 100644 --- a/cmdline/memory.c +++ b/cmdline/memory.c @@ -97,18 +97,6 @@ void* malloc_nofail(size_t size) /* LCOV_EXCL_STOP */ } -#ifndef CHECKER /* Don't preinitialize when running for valgrind */ - /* - * Here we preinitialize the memory to ensure that the OS is really allocating it - * and not only reserving the addressable space. - * Otherwise we are risking that the OOM (Out Of Memory) killer in Linux will kill the process. - * Filling the memory doesn't ensure to disable OOM, but it increase a lot the chances to - * get a real error from malloc() instead than a process killed. - * Note that calloc() doesn't have the same effect. - */ - memset(ptr, 0xA5, size); -#endif - malloc_counter_inc(size); return ptr; @@ -123,7 +111,6 @@ void* nalloc_nofail(size_t count, size_t size) size *= count; - /* see the note in malloc_nofail() of why we don't use calloc() */ void* ptr = malloc(size ? size : 1); if (!ptr) { @@ -140,9 +127,21 @@ void* nalloc_nofail(size_t count, size_t size) void* calloc_nofail(size_t count, size_t size) { - void* ptr = nalloc_nofail(count, size); + if (size != 0 && count > SIZE_MAX / size) { + log_fatal(EINTERNAL, "Allocation size overflow\n"); + os_abort(); + } - memset(ptr, 0, count * size); + void* ptr = calloc(count ? count : 1, size ? size : 1); + + if (!ptr) { + /* LCOV_EXCL_START */ + malloc_fail(count * size); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + malloc_counter_inc(count * size); return ptr; } From 97419fb67f8ee459f87870b2dec0e8c24be80ee9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 21:02:13 +0200 Subject: [PATCH 18/51] Cache BLOCK_HASH_SIZE in local variable in block run read and write loops --- cmdline/state.c | 78 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/cmdline/state.c b/cmdline/state.c index a21e96b4..83144634 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2298,11 +2298,11 @@ __aligned_loops(64) static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count, unsigned state, int discard_hash) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); - size_t block_stride = block_sizeof(); + size_t hash_size = BLOCK_HASH_SIZE; while (count) { size_t available = (size_t)(f->end - f->pos); - block_off_t cached_count = available / BLOCK_HASH_SIZE; + block_off_t cached_count = available / hash_size; unsigned char* input; if (cached_count > count) @@ -2313,33 +2313,50 @@ static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off int ret; block_state_set(block, state); - ret = sread(f, block->hash, BLOCK_HASH_SIZE); + ret = sread(f, block->hash, hash_size); if (ret < 0) return -1; if (discard_hash) hash_invalid_set(block->hash); - block_ptr += block_stride; + block_ptr += 1 + hash_size; --count; continue; } + count -= cached_count; input = f->pos; - for (block_off_t i = 0; i < cached_count; ++i) { - struct snapraid_block* block = (struct snapraid_block*)block_ptr; + if (hash_size == HASH_MAX) { + while (cached_count != 0) { + struct snapraid_block* block = (struct snapraid_block*)block_ptr; - block_state_set(block, state); - if (discard_hash) - hash_invalid_set(block->hash); - else - hash_copy(block->hash, input); + block_state_set(block, state); + if (discard_hash) + memset(block->hash, 0, HASH_MAX); + else + memcpy(block->hash, input, HASH_MAX); - input += BLOCK_HASH_SIZE; - block_ptr += block_stride; + input += HASH_MAX; + block_ptr += 1 + HASH_MAX; + --cached_count; + } + } else { + while (cached_count != 0) { + struct snapraid_block* block = (struct snapraid_block*)block_ptr; + + block_state_set(block, state); + if (discard_hash) + hash_invalid_set(block->hash); + else + hash_copy(block->hash, input); + + input += hash_size; + block_ptr += 1 + hash_size; + --cached_count; + } } f->pos = input; - count -= cached_count; } return 0; @@ -3907,11 +3924,11 @@ __aligned_loops(64) static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); - size_t block_stride = block_sizeof(); + size_t hash_size = BLOCK_HASH_SIZE; while (count) { size_t available = (size_t)(f->end - f->pos); - block_off_t cached_count = available / BLOCK_HASH_SIZE; + block_off_t cached_count = available / hash_size; unsigned char* output; if (cached_count > count) @@ -3920,26 +3937,39 @@ static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_of if (cached_count == 0) { struct snapraid_block* block = (struct snapraid_block*)block_ptr; - if (swrite(block->hash, BLOCK_HASH_SIZE, f) != 0) + if (swrite(block->hash, hash_size, f) != 0) return -1; - block_ptr += block_stride; + block_ptr += 1 + hash_size; --count; continue; } + count -= cached_count; output = f->pos; - for (block_off_t i = 0; i < cached_count; ++i) { - struct snapraid_block* block = (struct snapraid_block*)block_ptr; + if (hash_size == HASH_MAX) { + while (cached_count != 0) { + struct snapraid_block* block = (struct snapraid_block*)block_ptr; - hash_copy(output, block->hash); + memcpy(output, block->hash, HASH_MAX); - output += BLOCK_HASH_SIZE; - block_ptr += block_stride; + output += HASH_MAX; + block_ptr += 1 + HASH_MAX; + --cached_count; + } + } else { + while (cached_count != 0) { + struct snapraid_block* block = (struct snapraid_block*)block_ptr; + + hash_copy(output, block->hash); + + output += hash_size; + block_ptr += 1 + hash_size; + --cached_count; + } } f->pos = output; - count -= cached_count; } return 0; From 969cb5ca31b9917e838b89500550e3f540f84b87 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 16:50:31 +0200 Subject: [PATCH 19/51] Make sgetbs() return the number of characters read on success --- cmdline/mkstream.c | 2 +- cmdline/selftest.c | 24 ++++++++++++++++++++++++ cmdline/state.c | 2 +- cmdline/stream.c | 10 ++++++++-- cmdline/stream.h | 2 +- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/cmdline/mkstream.c b/cmdline/mkstream.c index 224e8380..146110fe 100644 --- a/cmdline/mkstream.c +++ b/cmdline/mkstream.c @@ -144,7 +144,7 @@ void test(int advise_mode) char copy[STR_MAX]; memset(str, ' ' + j, j - 1); str[j - 1] = 0; - if (sgetbs(s, copy, sizeof(copy)) != 0 || strcmp(copy, str) != 0) { + if (sgetbs(s, copy, sizeof(copy)) != (ssize_t)(j - 1) || strcmp(copy, str) != 0) { /* LCOV_EXCL_START */ exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ diff --git a/cmdline/selftest.c b/cmdline/selftest.c index a88f61e5..0aae3f3b 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -2183,6 +2183,30 @@ static void test_stream(void) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } + + /* sgetbs test */ + unsigned char buf_bs[] = { 0x85, 'h', 'e', 'l', 'l', 'o' }; + char str_bs[16]; + memset(&f, 0, sizeof(f)); + f.pos = buf_bs; + f.end = buf_bs + sizeof(buf_bs); + if (sgetbs(&f, str_bs, sizeof(str_bs)) != 5 || strcmp(str_bs, "hello") != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "test_stream: sgetbs valid string failed\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* sgetbs buffer too small test */ + memset(&f, 0, sizeof(f)); + f.pos = buf_bs; + f.end = buf_bs + sizeof(buf_bs); + if (sgetbs(&f, str_bs, 5) >= 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "test_stream: sgetbs small buffer accepted\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } static void test_extent_run(void) diff --git a/cmdline/state.c b/cmdline/state.c index 83144634..a233b628 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2416,7 +2416,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S int has_info; int has_invalid_parity; char buffer[PATH_MAX]; - int ret; + ssize_t ret; tommy_array disk_mapping; uint32_t mapping_max; tommy_hashdyn bucket_hash; diff --git a/cmdline/stream.c b/cmdline/stream.c index a7c49e33..dd978473 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -691,7 +691,7 @@ int sgetble32(STREAM* f, uint32_t* value) return 0; } -int sgetbs(STREAM* f, char* str, size_t size) +ssize_t sgetbs(STREAM* f, char* str, size_t size) { uint64_t len; @@ -709,7 +709,13 @@ int sgetbs(STREAM* f, char* str, size_t size) str[len] = 0; - return sread(f, str, len); + if (sread(f, str, len) != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + return len; } int swrite_uncached(const void* void_data, size_t size, STREAM* f) diff --git a/cmdline/stream.h b/cmdline/stream.h index ea9d52f6..08377adc 100644 --- a/cmdline/stream.h +++ b/cmdline/stream.h @@ -329,7 +329,7 @@ int sgetble32(STREAM* f, uint32_t* value); * Read a binary string. * Return -1 on error or if the buffer is too small, or the number of chars read. */ -int sgetbs(STREAM* f, char* str, size_t size); +ssize_t sgetbs(STREAM* f, char* str, size_t size); /****************************************************************************/ /* put */ From 3d7a722ab2d120f898bf031a6217f22ff983f36a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 17:00:24 +0200 Subject: [PATCH 20/51] Avoid redundant strlen and duplicate traversals when loading content strings --- cmdline/elem.c | 18 ++++++++-------- cmdline/elem.h | 54 +++++++++++++++++++++++++++++++++++++++++------- cmdline/memory.c | 10 +++++++++ cmdline/memory.h | 6 ++++++ cmdline/state.c | 36 ++++++++++++++++++++++++-------- 5 files changed, 99 insertions(+), 25 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 5d9f1108..2401c755 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -470,7 +470,7 @@ int filter_snapshot(int enable, const char* sub, const char* name) return 0; } -struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) +struct snapraid_file* file_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) { struct snapraid_file* file; block_off_t blockmax; @@ -489,7 +489,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ } file = malloc_nofail(sizeof(struct snapraid_file)); - file->sub = strdup_nofail(sub); + file->sub = strndup_nofail(sub, sub_len); file->size = size; file->blockmax = blockmax; file->mtime_sec = mtime_sec; @@ -794,13 +794,13 @@ int extent_file_compare(const void* void_a, const void* void_b) return 0; } -struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag) +struct snapraid_link* link_alloc_len(const char* sub, size_t sub_len, const char* linkto, size_t linkto_len, unsigned link_flag) { struct snapraid_link* slink; slink = malloc_nofail(sizeof(struct snapraid_link)); - slink->sub = strdup_nofail(sub); - slink->linkto = strdup_nofail(linkto); + slink->sub = strndup_nofail(sub, sub_len); + slink->linkto = strndup_nofail(linkto, linkto_len); slink->flag = link_flag; return slink; @@ -831,12 +831,12 @@ int link_alpha_compare(const void* void_a, const void* void_b) return strcmp(slink_a->sub, slink_b->sub); } -struct snapraid_dir* dir_alloc(const char* sub) +struct snapraid_dir* dir_alloc_len(const char* sub, size_t sub_len) { struct snapraid_dir* dir; dir = malloc_nofail(sizeof(struct snapraid_dir)); - dir->sub = strdup_nofail(sub); + dir->sub = strndup_nofail(sub, sub_len); dir->flag = 0; return dir; @@ -858,7 +858,7 @@ int dir_name_compare(const void* void_arg, const void* void_data) return strcmp(arg, dir->sub); } -struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec) +struct snapraid_dealloc* dealloc_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec) { struct snapraid_dealloc* dealloc; block_off_t blockmax; @@ -877,7 +877,7 @@ struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, dat } dealloc = malloc_nofail(sizeof(struct snapraid_dealloc) + blockmax * sizeof(struct snapraid_dealloc_block)); - dealloc->sub = strdup_nofail(sub); + dealloc->sub = strndup_nofail(sub, sub_len); dealloc->size = size; dealloc->mtime_sec = mtime_sec; dealloc->mtime_nsec = mtime_nsec; diff --git a/cmdline/elem.h b/cmdline/elem.h index 81dd62a7..a507ba58 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -908,7 +908,11 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask) /** * Allocate a file without initializing its blocks. */ -struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode); +struct snapraid_file* file_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode); +static inline struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) +{ + return file_alloc_len(block_size, sub, strlen(sub), size, mtime_sec, mtime_nsec, inode); +} /** * Duplicate a file. @@ -994,12 +998,20 @@ int file_stamp_compare(const void* void_a, const void* void_b); int file_namestamp_compare(const void* void_a, const void* void_b); int file_pathstamp_compare(const void* void_a, const void* void_b); +/** + * Compute the hash of a file path with known length. + */ +static inline tommy_uint32_t file_path_hash_len(const char* sub, size_t len) +{ + return tommy_hash_u32(0, sub, len); +} + /** * Compute the hash of a file path. */ static inline tommy_uint32_t file_path_hash(const char* sub) { - return tommy_hash_u32(0, sub, strlen(sub)); + return file_path_hash_len(sub, strlen(sub)); } /** @@ -1059,7 +1071,11 @@ static inline unsigned link_flag_get(struct snapraid_link* slink, unsigned mask) /** * Allocate a link. */ -struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag); +struct snapraid_link* link_alloc_len(const char* name, size_t name_len, const char* slink, size_t slink_len, unsigned link_flag); +static inline struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag) +{ + return link_alloc_len(name, strlen(name), slink, strlen(slink), link_flag); +} /** * Deallocate a link. @@ -1076,12 +1092,20 @@ int link_name_compare_to_arg(const void* void_arg, const void* void_data); */ int link_alpha_compare(const void* void_a, const void* void_b); +/** + * Compute the hash of a link name with known length. + */ +static inline tommy_uint32_t link_name_hash_len(const char* name, size_t len) +{ + return tommy_hash_u32(0, name, len); +} + /** * Compute the hash of a link name. */ static inline tommy_uint32_t link_name_hash(const char* name) { - return tommy_hash_u32(0, name, strlen(name)); + return link_name_hash_len(name, strlen(name)); } static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask) @@ -1102,7 +1126,11 @@ static inline void dir_flag_clear(struct snapraid_dir* dir, unsigned mask) /** * Allocate a dir. */ -struct snapraid_dir* dir_alloc(const char* name); +struct snapraid_dir* dir_alloc_len(const char* name, size_t name_len); +static inline struct snapraid_dir* dir_alloc(const char* name) +{ + return dir_alloc_len(name, strlen(name)); +} /** * Deallocate a dir. @@ -1114,18 +1142,30 @@ void dir_free(void* void_dir); */ int dir_name_compare(const void* void_arg, const void* void_data); +/** + * Compute the hash of a dir name with known length. + */ +static inline tommy_uint32_t dir_name_hash_len(const char* name, size_t len) +{ + return tommy_hash_u32(0, name, len); +} + /** * Compute the hash of a dir name. */ static inline tommy_uint32_t dir_name_hash(const char* name) { - return tommy_hash_u32(0, name, strlen(name)); + return dir_name_hash_len(name, strlen(name)); } /** * Allocate a dealloc. */ -struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec); +struct snapraid_dealloc* dealloc_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec); +static inline struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec) +{ + return dealloc_alloc_len(block_size, sub, strlen(sub), size, mtime_sec, mtime_nsec); +} /** * Deallocate a dealloc. diff --git a/cmdline/memory.c b/cmdline/memory.c index 3354e2d3..5a005fff 100644 --- a/cmdline/memory.c +++ b/cmdline/memory.c @@ -157,3 +157,13 @@ char* strdup_nofail(const char* str) return ptr; } +char* strndup_nofail(const char* str, size_t size) +{ + char* ptr = malloc_nofail(size + 1); + + memcpy(ptr, str, size); + ptr[size] = 0; + + return ptr; +} + diff --git a/cmdline/memory.h b/cmdline/memory.h index d2ed0b9b..76950e67 100644 --- a/cmdline/memory.h +++ b/cmdline/memory.h @@ -36,6 +36,12 @@ void* calloc_nofail(size_t count, size_t size); */ char* strdup_nofail(const char* str); +/** + * Safe strndup with known length. + * If no memory is available, it aborts. + */ +char* strndup_nofail(const char* str, size_t size); + /** * Helper for printing an error about a failed allocation. */ diff --git a/cmdline/state.c b/cmdline/state.c index a233b628..613ea606 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2596,6 +2596,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_STOP */ } + size_t sub_len; + ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { /* LCOV_EXCL_START */ @@ -2604,6 +2606,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + sub_len = ret; + if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2616,12 +2620,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S * Allocate without initializing blocks. The runs below cover the whole * file and set every state and hash before exposing the loaded state. */ - file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode); + file = file_alloc_len(state->block_size, sub, sub_len, v_size, v_mtime_sec, v_mtime_nsec, v_inode); /* insert the file in the file containers */ if (file->inode != INODE_INVALID) tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash_len(file->sub, sub_len)); tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); @@ -3015,6 +3019,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S uint64_t v_mtime_sec; uint32_t v_mtime_nsec; + size_t sub_len; + ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { /* LCOV_EXCL_START */ @@ -3023,6 +3029,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + sub_len = ret; + if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); @@ -3079,7 +3087,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S --v_mtime_nsec; /* allocate the file */ - struct snapraid_dealloc* dealloc = dealloc_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec); + struct snapraid_dealloc* dealloc = dealloc_alloc_len(state->block_size, sub, sub_len, v_size, v_mtime_sec, v_mtime_nsec); log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", disk->name, esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec); @@ -3124,6 +3132,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* symlink */ char sub[PATH_MAX]; char linkto[PATH_MAX]; + size_t sub_len; + size_t linkto_len; struct snapraid_link* slink; struct snapraid_disk* disk; uint32_t mapping; @@ -3146,6 +3156,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + sub_len = ret; if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ @@ -3163,12 +3174,13 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + linkto_len = ret; /* allocate the link as symbolic link */ - slink = link_alloc(sub, linkto, FILE_IS_SYMLINK); + slink = link_alloc_len(sub, sub_len, linkto, linkto_len, FILE_IS_SYMLINK); /* insert the link in the link containers */ - tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub)); + tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash_len(slink->sub, sub_len)); tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink); /* stat */ @@ -3177,6 +3189,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* hardlink */ char sub[PATH_MAX]; char linkto[PATH_MAX]; + size_t sub_len; + size_t linkto_len; struct snapraid_link* slink; struct snapraid_disk* disk; uint32_t mapping; @@ -3199,6 +3213,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + sub_len = ret; if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ @@ -3216,6 +3231,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + linkto_len = ret; if (!path_is_sub(linkto)) { /* LCOV_EXCL_START */ @@ -3226,10 +3242,10 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } /* allocate the link as hard link */ - slink = link_alloc(sub, linkto, FILE_IS_HARDLINK); + slink = link_alloc_len(sub, sub_len, linkto, linkto_len, FILE_IS_HARDLINK); /* insert the link in the link containers */ - tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub)); + tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash_len(slink->sub, sub_len)); tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink); /* stat */ @@ -3237,6 +3253,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } else if (c == 'r') { /* dir */ char sub[PATH_MAX]; + size_t sub_len; struct snapraid_dir* dir; struct snapraid_disk* disk; uint32_t mapping; @@ -3259,6 +3276,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S os_abort(); /* LCOV_EXCL_STOP */ } + sub_len = ret; if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ @@ -3269,10 +3287,10 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } /* allocate the dir */ - dir = dir_alloc(sub); + dir = dir_alloc_len(sub, sub_len); /* insert the dir in the dir containers */ - tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub)); + tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash_len(dir->sub, sub_len)); tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir); /* stat */ From 0d2f466b11b405c285627ae6e72fdcb59e465ec2 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 20:31:41 +0200 Subject: [PATCH 21/51] Count unsynced blocks using an extent event sweep instead of per-block lookups --- cmdline/selftest.c | 127 +++++++++++++++++++++++++++++++++++++ cmdline/state.c | 154 ++++++++++++++++++++++++++++++++++++++------- cmdline/state.h | 7 +++ 3 files changed, 264 insertions(+), 24 deletions(-) diff --git a/cmdline/selftest.c b/cmdline/selftest.c index 0aae3f3b..0072b318 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -2245,6 +2245,130 @@ static void test_extent_run(void) file_free(file); } +static void test_unsynced_sweep(void) +{ + struct snapraid_state state; + struct snapraid_disk* disk1; + struct snapraid_disk* disk2; + struct snapraid_file* file1; + struct snapraid_file* file2; + struct snapraid_file* file3; + struct snapraid_file* file4; + struct snapraid_file* del; + block_off_t i; + static const struct { + block_off_t blockmax; + uint64_t unsynced; + } bounds[] = { + { 0, 0 }, { 10, 0 }, { 15, 0 }, { 16, 1 }, { 19, 4 }, + { 20, 5 }, { 21, 5 }, { 22, 6 }, { 23, 7 }, { 29, 7 }, + { 30, 7 }, { 31, 8 }, { 32, 9 }, { 33, 10 }, { 34, 11 }, + { 100, 11 } + }; + + memset(&state, 0, sizeof(state)); + state.block_size = 4096; + tommy_list_init(&state.disklist); + + if (fs_count_unsynced(&state, 100) != 0) { + log_fatal(EINTERNAL, "test_unsynced_sweep: no disks failed\n"); + exit(EXIT_FAILURE); + } + + disk1 = disk_alloc("d1", "", 0, "", 0); + disk2 = disk_alloc("d2", "", 0, "", 0); + tommy_list_insert_tail(&state.disklist, &disk1->node, disk1); + tommy_list_insert_tail(&state.disklist, &disk2->node, disk2); + + /* initially both disks are empty */ + if (fs_count_unsynced(&state, 100) != 0) { + log_fatal(EINTERNAL, "test_unsynced_sweep: empty disks failed\n"); + exit(EXIT_FAILURE); + } + + /* file1 on disk1: 10 blocks at parity 10..19 (first 5 BLK, next 5 CHG) */ + file1 = file_alloc(4096, "f1", 10 * 4096, 0, 0, 0); + for (i = 0; i < 5; ++i) + block_state_set(file_block(file1, i), BLOCK_STATE_BLK); + for (i = 5; i < 10; ++i) + block_state_set(file_block(file1, i), BLOCK_STATE_CHG); + fs_allocate(disk1, 10, file1, 0, 10); + + /* file2 on disk2: 20 blocks at parity 0..19 (all BLK) */ + file2 = file_alloc(4096, "f2", 20 * 4096, 0, 0, 0); + for (i = 0; i < 20; ++i) + block_state_set(file_block(file2, i), BLOCK_STATE_BLK); + fs_allocate(disk2, 0, file2, 0, 20); + + /* parity 0..14 has only BLK or EMPTY; parity 15..19 has CHG on disk1 and BLK on disk2 -> 5 unsynced */ + if (fs_count_unsynced(&state, 20) != 5) { + log_fatal(EINTERNAL, "test_unsynced_sweep: mixed block run failed\n"); + exit(EXIT_FAILURE); + } + + /* deleted run on disk1: 3 blocks at parity 20..22 */ + del = file_alloc(4096, "", 3 * 4096, 0, 0, 0); + for (i = 0; i < 3; ++i) + block_state_set(file_block(del, i), BLOCK_STATE_DELETED); + fs_allocate(disk1, 20, del, 0, 3); + + /* disk2 is empty at 20..22 -> DELETED alone does not count as unsynced */ + if (fs_count_unsynced(&state, 23) != 5) { + log_fatal(EINTERNAL, "test_unsynced_sweep: deleted without other file failed\n"); + exit(EXIT_FAILURE); + } + + /* file3 on disk2: first 2 blocks at parity 21..22 (BLK) */ + file3 = file_alloc(4096, "f3", 6 * 4096, 0, 0, 0); + for (i = 0; i < 2; ++i) + block_state_set(file_block(file3, i), BLOCK_STATE_BLK); + block_state_set(file_block(file3, 2), BLOCK_STATE_REP); + block_state_set(file_block(file3, 3), BLOCK_STATE_REP); + block_state_set(file_block(file3, 4), BLOCK_STATE_BLK); + block_state_set(file_block(file3, 5), BLOCK_STATE_REBUILD); + fs_allocate(disk2, 21, file3, 0, 2); + + /* parity 20: DELETED on d1, empty on d2 -> not unsynced; + * parity 21..22: DELETED on d1, BLK on d2 -> unsynced (2 blocks). Total 5 + 2 = 7 */ + if (fs_count_unsynced(&state, 23) != 7) { + log_fatal(EINTERNAL, "test_unsynced_sweep: deleted with file failed\n"); + exit(EXIT_FAILURE); + } + + /* the second extent of file3 starts at a nonzero file offset after a gap */ + fs_allocate(disk2, 30, file3, 2, 4); + + /* file4 shares extent boundaries with file3 and changes flags at 32 and 33 */ + file4 = file_alloc(4096, "f4", 4 * 4096, 0, 0, 0); + for (i = 0; i < 4; ++i) + block_state_set(file_block(file4, i), BLOCK_STATE_BLK); + block_state_set(file_block(file4, 2), BLOCK_STATE_CHG); + fs_allocate(disk1, 30, file4, 0, 4); + + /** + * Positions 30..33 are all unsynced: REP, REP, CHG, REBUILD respectively. + * At 32 and 33 the invalid block changes disks at the same event boundary; + * count each position once using the refreshed flags from both disks. + * Check limits in gaps, within extents and runs, and at their boundaries. + */ + for (i = 0; i < sizeof(bounds) / sizeof(bounds[0]); ++i) { + uint64_t count = fs_count_unsynced(&state, bounds[i].blockmax); + if (count != bounds[i].unsynced) { + log_fatal(EINTERNAL, "test_unsynced_sweep: limit %" PRIu64 " expected %" PRIu64 ", got %" PRIu64 "\n", + bounds[i].blockmax, bounds[i].unsynced, count); + exit(EXIT_FAILURE); + } + } + + file_free(file1); + file_free(file2); + file_free(file3); + file_free(file4); + file_free(del); + disk_free(disk1); + disk_free(disk2); +} + static void test_raid(void) { /* vandermonde raid parity generation with 32 data disks */ @@ -2965,6 +3089,9 @@ void test(int argc, char* argv[]) /* extent-aware file-to-parity run lookup */ test_extent_run(); + /* extent sweep for unsynced parity blocks */ + test_unsynced_sweep(); + /* wildcard and path pattern matching */ test_wnmatch(); diff --git a/cmdline/state.c b/cmdline/state.c index 613ea606..aca4f3ec 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2197,31 +2197,139 @@ static inline int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t po } /** - * Check if a block parity needs sync - * - * This is the same logic used in "status" to detect an incomplete "sync", - * that ignores invalid block, if they are not used by a file in any disk. - * - * This means that DELETED blocks won't necessarily imply an invalid parity. + * Advance a disk cursor to pos and cache its flags and next event boundary. + * Events occur at extent boundaries and changes in block flags within an extent. */ -static int fs_is_block_unsynced(struct snapraid_state* state, block_off_t pos) +static inline void disk_advance( + tommy_tree_node** cursor_ptr, + block_off_t pos, + block_off_t blockmax, + int* has_file_ptr, + int* has_invalid_ptr, + block_off_t* next_event_ptr, + size_t stride +) { - int one_invalid = 0; - int one_valid = 0; - for (tommy_node* i = state->disklist; i != 0; i = i->next) { - struct snapraid_disk* disk = i->data; - struct snapraid_block* block = fs_par2block_find(disk, pos); - - if (block_has_file(block)) - one_valid = 1; - if (block_has_invalid_parity(block)) - one_invalid = 1; - - if (one_invalid && one_valid) - return 1; + /* advance cursor if the current extent ends at or before pos */ + while (*cursor_ptr != 0) { + struct snapraid_extent* ext = (*cursor_ptr)->data; + if (ext->parity_pos + ext->count > pos) + break; + *cursor_ptr = tommy_tree_next(*cursor_ptr); } - return 0; + if (*cursor_ptr == 0) { + /* no more extents on this disk */ + *has_file_ptr = 0; + *has_invalid_ptr = 0; + *next_event_ptr = blockmax; + return; + } + + struct snapraid_extent* ext = (*cursor_ptr)->data; + if (ext->parity_pos > pos) { + /* in a gap before the next extent */ + *has_file_ptr = 0; + *has_invalid_ptr = 0; + *next_event_ptr = ext->parity_pos; + } else { + /* pos is inside this extent */ + block_off_t extent_rem = ext->parity_pos + ext->count - pos; + block_off_t file_pos = ext->file_pos + (pos - ext->parity_pos); + struct snapraid_block* block = file_block(ext->file, file_pos); + int has_file = block_has_file(block); + int has_invalid = block_has_invalid_parity(block); + block_off_t run_rem = 1; + const unsigned char* ptr = (const unsigned char*)block; + + *has_file_ptr = has_file; + *has_invalid_ptr = has_invalid; + + /* find how many consecutive blocks on this disk share the same flags */ + while (run_rem < extent_rem) { + const struct snapraid_block* next_b = (const struct snapraid_block*)(ptr + run_rem * stride); + if (block_has_file(next_b) != has_file || block_has_invalid_parity(next_b) != has_invalid) + break; + ++run_rem; + } + + *next_event_ptr = pos + run_rem; + } +} + +/** + * Count the number of unsynced parity positions using an event sweep across extents. + * + * An unsynced parity position is one where at least one disk has a live file + * block, and at least one disk has an invalid parity block (CHG, REP, REBUILD, + * or DELETED). + * + * The sweep takes O(B + R * D) time, where B is the number of blocks inspected, + * R is the number of event positions processed (including initialization), and + * D is the number of disks. Cached event boundaries avoid rescanning block runs. + * + * The caller must ensure that the disk list, extent trees, and block states + * remain unchanged during the count, as this function does not acquire locks. + */ +uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmax) +{ + tommy_tree_node* cursors[RAID_DATA_MAX]; + block_off_t next_events[RAID_DATA_MAX]; + int disk_has_file[RAID_DATA_MAX]; + int disk_has_invalid[RAID_DATA_MAX]; + unsigned disk_count = 0; + tommy_node* node; + uint64_t total_unsynced = 0; + block_off_t pos = 0; + size_t stride = block_sizeof(); + unsigned d; + + /* state_map() validates the disk limit only after content loading */ + if (tommy_list_count(&state->disklist) > RAID_DATA_MAX) { + /* LCOV_EXCL_START */ + log_fatal(ESOFT, "Too many data disks. Maximum allowed is %u.\n", RAID_DATA_MAX); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + for (node = state->disklist; node != 0; node = node->next) { + struct snapraid_disk* disk = node->data; + cursors[disk_count] = tommy_tree_head(&disk->fs_parity); + /* force an initial event so flags are set before their first use */ + next_events[disk_count] = 0; + ++disk_count; + } + + while (pos < blockmax) { + block_off_t next_pos = blockmax; + int one_file = 0; + int one_invalid = 0; + + for (d = 0; d < disk_count; ++d) { + /* refresh expired runs before counting the interval starting at pos */ + if (next_events[d] == pos) { + disk_advance( + &cursors[d], pos, blockmax, + &disk_has_file[d], &disk_has_invalid[d], + &next_events[d], stride + ); + } + + if (disk_has_file[d]) + one_file = 1; + if (disk_has_invalid[d]) + one_invalid = 1; + if (next_events[d] < next_pos) + next_pos = next_events[d]; + } + + if (one_file && one_invalid) + total_unsynced += (next_pos - pos); + + pos = next_pos; + } + + return total_unsynced; } /** @@ -2880,9 +2988,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S * count is certainly zero. Scan the extent maps only when needed. */ if (has_invalid_parity) { - for (v_pos = 0; v_pos < blockmax; ++v_pos) - if (fs_is_block_unsynced(state, v_pos)) - ++count_unsynced; + count_unsynced = fs_count_unsynced(state, blockmax); } } else if (c == 'h') { /* hole */ diff --git a/cmdline/state.h b/cmdline/state.h index 993466a7..441e1415 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -240,6 +240,13 @@ void state_probe(struct snapraid_state* state); */ void state_read(struct snapraid_state* state); +/** + * Count the number of unsynced parity positions using an event sweep across extents. + * Counts positions in [0, blockmax). The caller must ensure that the disk list, + * extent trees, and block states remain unchanged for the duration of the call. + */ +uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmax); + /** * Write the new state. */ From 93c39d5526cbaed0c6495db5432087e22a2521b2 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 21:11:09 +0200 Subject: [PATCH 22/51] Add tommy_hashdyn_reserve to pre-allocate hash buckets for expected elements --- tommyds/tommyhashdyn.c | 72 ++++++++++++++++++++++++++++-------------- tommyds/tommyhashdyn.h | 10 ++++++ 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/tommyds/tommyhashdyn.c b/tommyds/tommyhashdyn.c index ce2587fb..54d213f2 100644 --- a/tommyds/tommyhashdyn.c +++ b/tommyds/tommyhashdyn.c @@ -28,36 +28,47 @@ TOMMY_API void tommy_hashdyn_done(tommy_hashdyn* hashdyn) */ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket_bit) { - tommy_size_t bucket_bit; - tommy_size_t bucket_max; - tommy_size_t new_bucket_max; - tommy_size_t new_bucket_mask; + tommy_size_t bucket_bit = hashdyn->bucket_bit; + tommy_size_t bucket_max = hashdyn->bucket_max; + tommy_size_t new_bucket_max = (tommy_size_t)1 << new_bucket_bit; + tommy_size_t new_bucket_mask = new_bucket_max - 1; tommy_hashdyn_node** new_bucket; - bucket_bit = hashdyn->bucket_bit; - bucket_max = hashdyn->bucket_max; - - new_bucket_max = (tommy_size_t)1 << new_bucket_bit; - new_bucket_mask = new_bucket_max - 1; - - /* allocate the new vector using malloc() and not calloc() */ - /* because data is fully initialized in the update process */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); - - /* reinsert all the elements */ - if (new_bucket_bit > bucket_bit) { + if (!hashdyn->count) { + /* empty table: allocate zeroed memory with calloc without looping */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); + } else if (new_bucket_bit == bucket_bit + 1) { tommy_size_t i; - /* grow */ + /* grow by 1 bit: zero the two buckets inline in loop to preserve cache locality */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); + for (i = 0; i < bucket_max; ++i) { tommy_hashdyn_node* j; - /* setup the new two buckets */ new_bucket[i] = 0; new_bucket[i + bucket_max] = 0; - /* reinsert the bucket */ j = hashdyn->bucket[i]; + while (j) { + tommy_hashdyn_node* j_next = j->next; + tommy_size_t pos = j->index & new_bucket_mask; + if (new_bucket[pos]) + tommy_list_insert_tail_not_empty(new_bucket[pos], j); + else + tommy_list_insert_first(&new_bucket[pos], j); + j = j_next; + } + } + } else if (new_bucket_bit > bucket_bit) { + tommy_size_t i; + + /* grow by multiple bits with elements: allocate zeroed memory and reinsert */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); + + for (i = 0; i < bucket_max; ++i) { + tommy_hashdyn_node* j = hashdyn->bucket[i]; + while (j) { tommy_hashdyn_node* j_next = j->next; tommy_size_t pos = j->index & new_bucket_mask; @@ -71,12 +82,11 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket } else { tommy_size_t i; - /* shrink */ - for (i = 0; i < new_bucket_max; ++i) { - /* setup the new bucket with the lower bucket*/ - new_bucket[i] = hashdyn->bucket[i]; + /* shrink with elements: all buckets are overwritten, no pre-zeroing needed */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); - /* concat the upper bucket */ + for (i = 0; i < new_bucket_max; ++i) { + new_bucket[i] = hashdyn->bucket[i]; tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); } } @@ -110,6 +120,19 @@ tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn) tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); } +TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count) +{ + tommy_uint_t bucket_bit = tommy_ilog2(count | 1) + 2; + + if (bucket_bit < TOMMY_HASHDYN_BIT) + bucket_bit = TOMMY_HASHDYN_BIT; + + if (bucket_bit <= hashdyn->bucket_bit) + return; + + tommy_hashdyn_resize(hashdyn, bucket_bit); +} + TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) { tommy_size_t pos = hash & hashdyn->bucket_mask; @@ -256,3 +279,4 @@ TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list) memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); hashdyn->count = 0; } + diff --git a/tommyds/tommyhashdyn.h b/tommyds/tommyhashdyn.h index 5ed7ed7a..9bce46be 100644 --- a/tommyds/tommyhashdyn.h +++ b/tommyds/tommyhashdyn.h @@ -155,6 +155,15 @@ TOMMY_API void tommy_hashdyn_init(tommy_hashdyn* hashdyn); */ TOMMY_API void tommy_hashdyn_done(tommy_hashdyn* hashdyn); +/** + * Pre-allocates buckets for the specified number of elements. + * + * It ensures that the hashtable has enough buckets allocated to contain + * the specified number of elements without triggering a dynamic resize. + * \param count Number of elements to reserve space for. + */ +TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count); + /** * Inserts an element in the hashtable. */ @@ -308,3 +317,4 @@ TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn); TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list); #endif + From 85893271144da47db7e941e24f0098916cbd5375 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 21:19:56 +0200 Subject: [PATCH 23/51] Defer file hash table insertions and pre-reserve bucket capacity in state_read_content --- cmdline/state.c | 43 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/cmdline/state.c b/cmdline/state.c index aca4f3ec..9f504e21 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2527,6 +2527,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S ssize_t ret; tommy_array disk_mapping; uint32_t mapping_max; + tommy_size_t disk_file_count[RAID_DATA_MAX]; tommy_hashdyn bucket_hash; blockmax = 0; @@ -2547,6 +2548,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S has_info = 0; has_invalid_parity = 0; mapping_max = 0; + memset(disk_file_count, 0, sizeof(disk_file_count)); tommy_array_init(&disk_mapping); tommy_hashdyn_init(&bucket_hash); @@ -2730,12 +2732,13 @@ static void state_read_content(struct snapraid_state* state, const char* path, S */ file = file_alloc_len(state->block_size, sub, sub_len, v_size, v_mtime_sec, v_mtime_nsec, v_inode); - /* insert the file in the file containers */ - if (file->inode != INODE_INVALID) - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash_len(file->sub, sub_len)); - tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); + /* + * Append the file to the disk list. Hash table insertions into + * inodeset, pathset, and stampset are deferred until all records + * are parsed to avoid cache thrashing and allow pre-reserving buckets. + */ tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); + ++disk_file_count[mapping]; /* read all the blocks */ v_idx = 0; @@ -3923,6 +3926,36 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } } + /* + * Reserve hash table bucket capacity upfront and populate containers for + * all loaded files. Deferring this until after stream parsing avoids cache + * thrashing during I/O and ensures zero dynamic resizes in tommy_hashdyn. + */ + for (uint32_t i = 0; i < mapping_max; ++i) { + struct snapraid_disk* disk = tommy_array_get(&disk_mapping, i); + tommy_size_t count = disk_file_count[i]; + tommy_node* node; + + if (count == 0) + continue; + + tommy_hashdyn_reserve(&disk->pathset, count); + tommy_hashdyn_reserve(&disk->inodeset, count); + tommy_hashdyn_reserve(&disk->stampset, count); + + node = tommy_list_head(&disk->filelist); + while (node) { + struct snapraid_file* file = node->data; + + if (file->inode != INODE_INVALID) + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); + + node = node->next; + } + } + tommy_array_done(&disk_mapping); if (serror(f)) { From 9f24ca00d1e6af761f628e70d50c572b86df999e Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 17:28:26 +0200 Subject: [PATCH 24/51] Use atomic operations instead of mutex for memory allocation counter --- cmdline/memory.c | 49 +++++++++++++++++++++++++++++++---------------- cmdline/memory.h | 5 +++++ cmdline/state.c | 3 ++- cmdline/support.c | 17 ---------------- cmdline/support.h | 6 ------ 5 files changed, 40 insertions(+), 40 deletions(-) diff --git a/cmdline/memory.c b/cmdline/memory.c index 5a005fff..f2c395a3 100644 --- a/cmdline/memory.c +++ b/cmdline/memory.c @@ -14,28 +14,45 @@ */ static size_t mcounter; +#if HAVE_THREAD +static int memory_is_single_thread; + +void memory_single_thread(int single_thread) +{ + memory_is_single_thread = single_thread; +} + +static inline void malloc_counter_inc(size_t inc) +{ + if (tommy_likely(memory_is_single_thread)) + mcounter += inc; + else + __atomic_fetch_add(&mcounter, inc, __ATOMIC_RELAXED); +} + size_t malloc_counter_get(void) { - size_t ret; - - lock_memory(); - - ret = mcounter; - - unlock_memory(); - - return ret; + if (tommy_likely(memory_is_single_thread)) + return mcounter; + return __atomic_load_n(&mcounter, __ATOMIC_RELAXED); } - -static void malloc_counter_inc(size_t inc) +#else +void memory_single_thread(int single_thread) { - lock_memory(); - - mcounter += inc; - - unlock_memory(); + (void)single_thread; } +static inline void malloc_counter_inc(size_t inc) +{ + mcounter += inc; +} + +size_t malloc_counter_get(void) +{ + return mcounter; +} +#endif + /* LCOV_EXCL_START */ static ssize_t malloc_print(int f, const char* str) { diff --git a/cmdline/memory.h b/cmdline/memory.h index 76950e67..0d05f376 100644 --- a/cmdline/memory.h +++ b/cmdline/memory.h @@ -12,6 +12,11 @@ */ size_t malloc_counter_get(void); +/** + * Enable or disable single-thread mode for memory counters. + */ +void memory_single_thread(int single_thread); + /** * Safe malloc. * If no memory is available, it aborts. diff --git a/cmdline/state.c b/cmdline/state.c index 9f504e21..c9c83ff0 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2477,6 +2477,8 @@ static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off */ static void fs_single_thread(struct snapraid_state* state, int single_thread) { + memory_single_thread(single_thread); + #if HAVE_THREAD for (tommy_node* i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; @@ -2484,7 +2486,6 @@ static void fs_single_thread(struct snapraid_state* state, int single_thread) } #else (void)state; - (void)single_thread; #endif } diff --git a/cmdline/support.c b/cmdline/support.c index ae01598c..bbb91d7d 100644 --- a/cmdline/support.c +++ b/cmdline/support.c @@ -52,7 +52,6 @@ char* esc_buf(void) */ #if HAVE_THREAD static thread_mutex_t msg_lock; -static thread_mutex_t memory_lock; static thread_mutex_t random_lock; #endif @@ -70,20 +69,6 @@ void unlock_msg(void) #endif } -void lock_memory(void) -{ -#if HAVE_THREAD - thread_mutex_lock(&memory_lock); -#endif -} - -void unlock_memory(void) -{ -#if HAVE_THREAD - thread_mutex_unlock(&memory_lock); -#endif -} - void lock_random(void) { #if HAVE_THREAD @@ -103,7 +88,6 @@ void lock_init(void) #if HAVE_THREAD /* initialize the locks as first operation as log_fatal depends on them */ thread_mutex_init(&msg_lock); - thread_mutex_init(&memory_lock); thread_mutex_init(&random_lock); thread_key_create(&esc_key, esc_pool_free); #endif @@ -119,7 +103,6 @@ void lock_done(void) } thread_key_delete(esc_key); thread_mutex_destroy(&msg_lock); - thread_mutex_destroy(&memory_lock); thread_mutex_destroy(&random_lock); #endif } diff --git a/cmdline/support.h b/cmdline/support.h index 995901f8..34f945b0 100644 --- a/cmdline/support.h +++ b/cmdline/support.h @@ -36,12 +36,6 @@ void lock_done(void); void lock_msg(void); void unlock_msg(void); -/** - * Lock used for memory counter. - */ -void lock_memory(void); -void unlock_memory(void); - /****************************************************************************/ /* random */ From 143813a90e0e8b0b81fe15a3bd34d47dc4640e98 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2026 17:12:44 +0200 Subject: [PATCH 25/51] Store initial file path inline with struct snapraid_file --- cmdline/elem.c | 21 +++++++++++++++------ cmdline/elem.h | 2 ++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 2401c755..ebf3e569 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -488,8 +488,10 @@ struct snapraid_file* file_alloc_len(unsigned block_size, const char* sub, size_ exit(EXIT_FAILURE); } - file = malloc_nofail(sizeof(struct snapraid_file)); - file->sub = strndup_nofail(sub, sub_len); + file = malloc_nofail(sizeof(struct snapraid_file) + sub_len + 1); + file->sub = file->sub_inline; + memcpy(file->sub, sub, sub_len); + file->sub[sub_len] = 0; file->size = size; file->blockmax = blockmax; file->mtime_sec = mtime_sec; @@ -505,6 +507,7 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) { struct snapraid_file* file; block_off_t i; + size_t sub_len; if (copy->blockmax > BLOCK_MAX) { #if SIZE_MAX == UINT32_MAX @@ -515,8 +518,11 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) exit(EXIT_FAILURE); } - file = malloc_nofail(sizeof(struct snapraid_file)); - file->sub = strdup_nofail(copy->sub); + sub_len = strlen(copy->sub); + file = malloc_nofail(sizeof(struct snapraid_file) + sub_len + 1); + file->sub = file->sub_inline; + memcpy(file->sub, copy->sub, sub_len); + file->sub[sub_len] = 0; file->size = copy->size; file->blockmax = copy->blockmax; file->mtime_sec = copy->mtime_sec; @@ -539,7 +545,8 @@ void file_free(void* void_file) { struct snapraid_file* file = void_file; - free(file->sub); + if (file->sub != file->sub_inline) + free(file->sub); file->sub = 0; free(file->blockvec); file->blockvec = 0; @@ -548,7 +555,9 @@ void file_free(void* void_file) void file_rename(struct snapraid_file* file, const char* sub) { - free(file->sub); + if (file->sub != file->sub_inline) + free(file->sub); + file->sub = strdup_nofail(sub); } diff --git a/cmdline/elem.h b/cmdline/elem.h index a507ba58..e767b024 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -341,6 +341,8 @@ struct snapraid_file { tommy_hashdyn_node nodeset; tommy_hashdyn_node pathset; tommy_hashdyn_node stampset; + + char sub_inline[]; /**< Inline storage for the initial sub path. */ }; /** From 6ae014d6ed7d641b7211be6d3f63e5107ddbdc11 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 5 Oct 2026 13:46:22 +0200 Subject: [PATCH 26/51] Align integer gen and rec loops to 64 bytes --- raid/NOTES | 6 ++++++ raid/int.c | 19 +++++++++++++++++++ raid/internal.h | 15 +++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/raid/NOTES b/raid/NOTES index 20028207..b095fc2a 100644 --- a/raid/NOTES +++ b/raid/NOTES @@ -1055,6 +1055,12 @@ The implementation deliberately targets at most six parity levels. Short list of ideas already tried or rejected --------------------------------------------- +* Loop alignment to 64 bytes: + improves scalar integer routines (raid/int.c) by +4% to +15% by avoiding fetch + and decode stalls on compact loops. Tried on vector SIMD routines (SSE, AVX, + NEON), with no improvement because wide unrolled loops are memory-bound and + cached in the uop cache. Kept only for integer routines. + * AVX512 128-byte generation processing: tried, can be slower. diff --git a/raid/int.c b/raid/int.c index 0aa5148e..9587c4d4 100644 --- a/raid/int.c +++ b/raid/int.c @@ -9,6 +9,7 @@ * * Uses 16-byte chunks across four 32-bit words. */ +__aligned_loops(64) void raid_gen1_int32(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -49,6 +50,7 @@ void raid_gen1_int32(int nd, size_t size, void **vv, int streaming) * * Uses 32-byte chunks across four 64-bit words. */ +__aligned_loops(64) void raid_gen1_int64(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -193,6 +195,7 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, * * Uses Horner's method with 8-byte chunks across two 32-bit words. */ +__aligned_loops(64) void raid_genz_int32_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -250,6 +253,7 @@ void raid_genz_int32_raid(int nd, size_t size, void **vv, int streaming) * * Uses Horner's method with 16-byte chunks across two 64-bit words. */ +__aligned_loops(64) void raid_genz_int64_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -307,6 +311,7 @@ void raid_genz_int64_raid(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ +__aligned_loops(64) void raid_gen2_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -361,6 +366,7 @@ void raid_gen2_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ +__aligned_loops(64) void raid_gen3_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -425,6 +431,7 @@ void raid_gen3_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ +__aligned_loops(64) void raid_gen4_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -498,6 +505,7 @@ void raid_gen4_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ +__aligned_loops(64) void raid_gen5_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -580,6 +588,7 @@ void raid_gen5_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ +__aligned_loops(64) void raid_gen6_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -1008,30 +1017,35 @@ static __always_inline void raid_recX_int8(int nr, int has_p, int *id, int *ip, } } +__aligned_loops(64) void raid_gen2_int32_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int32_gen(nd, size, vv, 2); } +__aligned_loops(64) void raid_gen2_int32_aes(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int32_gen(nd, size, vv, 3); } +__aligned_loops(64) void raid_gen2_int64_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int64_gen(nd, size, vv, 2); } +__aligned_loops(64) void raid_gen2_int64_aes(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int64_gen(nd, size, vv, 3); } +__aligned_loops(64) void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); @@ -1042,6 +1056,7 @@ void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(1, 0, id, ip, nd, size, vv); } +__aligned_loops(64) void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); @@ -1052,6 +1067,7 @@ void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(2, 0, id, ip, nd, size, vv); } +__aligned_loops(64) void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); @@ -1061,6 +1077,7 @@ void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(3, 0, id, ip, nd, size, vv); } +__aligned_loops(64) void raid_rec4_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); @@ -1070,6 +1087,7 @@ void raid_rec4_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(4, 0, id, ip, nd, size, vv); } +__aligned_loops(64) void raid_rec5_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); @@ -1079,6 +1097,7 @@ void raid_rec5_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(5, 0, id, ip, nd, size, vv); } +__aligned_loops(64) void raid_rec6_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); diff --git a/raid/internal.h b/raid/internal.h index 4fbe9b89..0314c95a 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -61,7 +61,22 @@ * Forced alignment. */ #ifndef __aligned +#if defined(__GNUC__) || defined(__clang__) #define __aligned(a) __attribute__((aligned(a))) +#else +#define __aligned(a) +#endif +#endif + +/* + * Forced loops alignment. + */ +#ifndef __aligned_loops +#if defined(__GNUC__) && !defined(__clang__) +#define __aligned_loops(n) __attribute__((optimize("align-loops=" #n))) +#else +#define __aligned_loops(n) +#endif #endif /* From 2a60279674d6ed3502869676a753401519818544 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 6 Oct 2026 09:17:23 +0200 Subject: [PATCH 27/51] Avoid strlen in deferred pathset insertion by pre-caching path hash in pathset.index --- cmdline/state.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmdline/state.c b/cmdline/state.c index c9c83ff0..81a41914 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2737,7 +2737,10 @@ static void state_read_content(struct snapraid_state* state, const char* path, S * Append the file to the disk list. Hash table insertions into * inodeset, pathset, and stampset are deferred until all records * are parsed to avoid cache thrashing and allow pre-reserving buckets. + * Cache the path hash in pathset.index to avoid recomputing strlen + * during insertion. */ + file->pathset.index = file_path_hash_len(file->sub, sub_len); tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); ++disk_file_count[mapping]; @@ -3950,7 +3953,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (file->inode != INODE_INVALID) tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file->pathset.index); tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); node = node->next; From 8fe9331f426c072e0e1ad6cee3b683f832aef20e Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 6 Oct 2026 10:42:03 +0200 Subject: [PATCH 28/51] Limit unsynced parity sweep to the invalid window tracked during content parsing --- cmdline/elem.c | 14 ++++++++++++ cmdline/elem.h | 7 ++++++ cmdline/selftest.c | 27 +++++++++++++--------- cmdline/state.c | 56 ++++++++++++++++++++++++++++++---------------- cmdline/state.h | 4 ++-- 5 files changed, 76 insertions(+), 32 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index ebf3e569..3ec737f7 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -1389,6 +1389,20 @@ static int extent_parity_inside_compare_unlock(const void* void_a, const void* v return 0; } +struct snapraid_extent* fs_par2extent_find_next(struct snapraid_disk* disk, block_off_t parity_pos) +{ + struct extent_parity_inside arg = { parity_pos }; + struct snapraid_extent* extent; + + fs_lock(disk); + + /* non-overlapping extents let this search find the containing extent or the next one */ + extent = tommy_tree_search_greater_equal_compare(&disk->fs_parity, extent_parity_inside_compare_unlock, &arg); + + fs_unlock(disk); + return extent; +} + /** * Search the extent at the specified parity position. * The search is optimized for sequential accesses. diff --git a/cmdline/elem.h b/cmdline/elem.h index e767b024..b9a638ae 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -1243,6 +1243,13 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t pos); */ struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos); +/** + * Find the first extent ending after the specified parity position. + * Return 0 if no such extent exists. The caller must ensure that the extent + * trees remain unchanged while using the returned extent or its tree node. + */ +struct snapraid_extent* fs_par2extent_find_next(struct snapraid_disk* disk, block_off_t parity_pos); + /** * Get the file position from the parity position. * Return 0 if no file is using it. diff --git a/cmdline/selftest.c b/cmdline/selftest.c index 0072b318..602f80e3 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -2270,7 +2270,7 @@ static void test_unsynced_sweep(void) state.block_size = 4096; tommy_list_init(&state.disklist); - if (fs_count_unsynced(&state, 100) != 0) { + if (fs_count_unsynced(&state, 0, 100) != 0) { log_fatal(EINTERNAL, "test_unsynced_sweep: no disks failed\n"); exit(EXIT_FAILURE); } @@ -2281,7 +2281,7 @@ static void test_unsynced_sweep(void) tommy_list_insert_tail(&state.disklist, &disk2->node, disk2); /* initially both disks are empty */ - if (fs_count_unsynced(&state, 100) != 0) { + if (fs_count_unsynced(&state, 10, 100) != 0) { log_fatal(EINTERNAL, "test_unsynced_sweep: empty disks failed\n"); exit(EXIT_FAILURE); } @@ -2301,7 +2301,7 @@ static void test_unsynced_sweep(void) fs_allocate(disk2, 0, file2, 0, 20); /* parity 0..14 has only BLK or EMPTY; parity 15..19 has CHG on disk1 and BLK on disk2 -> 5 unsynced */ - if (fs_count_unsynced(&state, 20) != 5) { + if (fs_count_unsynced(&state, 15, 20) != 5) { log_fatal(EINTERNAL, "test_unsynced_sweep: mixed block run failed\n"); exit(EXIT_FAILURE); } @@ -2313,7 +2313,7 @@ static void test_unsynced_sweep(void) fs_allocate(disk1, 20, del, 0, 3); /* disk2 is empty at 20..22 -> DELETED alone does not count as unsynced */ - if (fs_count_unsynced(&state, 23) != 5) { + if (fs_count_unsynced(&state, 15, 23) != 5) { log_fatal(EINTERNAL, "test_unsynced_sweep: deleted without other file failed\n"); exit(EXIT_FAILURE); } @@ -2330,7 +2330,7 @@ static void test_unsynced_sweep(void) /* parity 20: DELETED on d1, empty on d2 -> not unsynced; * parity 21..22: DELETED on d1, BLK on d2 -> unsynced (2 blocks). Total 5 + 2 = 7 */ - if (fs_count_unsynced(&state, 23) != 7) { + if (fs_count_unsynced(&state, 15, 23) != 7) { log_fatal(EINTERNAL, "test_unsynced_sweep: deleted with file failed\n"); exit(EXIT_FAILURE); } @@ -2349,14 +2349,19 @@ static void test_unsynced_sweep(void) * Positions 30..33 are all unsynced: REP, REP, CHG, REBUILD respectively. * At 32 and 33 the invalid block changes disks at the same event boundary; * count each position once using the refreshed flags from both disks. - * Check limits in gaps, within extents and runs, and at their boundaries. + * Check every window between limits in gaps, within extents and runs, + * and at their boundaries, including empty windows and nonzero starts. */ for (i = 0; i < sizeof(bounds) / sizeof(bounds[0]); ++i) { - uint64_t count = fs_count_unsynced(&state, bounds[i].blockmax); - if (count != bounds[i].unsynced) { - log_fatal(EINTERNAL, "test_unsynced_sweep: limit %" PRIu64 " expected %" PRIu64 ", got %" PRIu64 "\n", - bounds[i].blockmax, bounds[i].unsynced, count); - exit(EXIT_FAILURE); + block_off_t j; + for (j = 0; j <= i; ++j) { + uint64_t expected = bounds[i].unsynced - bounds[j].unsynced; + uint64_t count = fs_count_unsynced(&state, bounds[j].blockmax, bounds[i].blockmax); + if (count != expected) { + log_fatal(EINTERNAL, "test_unsynced_sweep: window [%" PRIu64 ", %" PRIu64 ") expected %" PRIu64 ", got %" PRIu64 "\n", + bounds[j].blockmax, bounds[i].blockmax, expected, count); + exit(EXIT_FAILURE); + } } } diff --git a/cmdline/state.c b/cmdline/state.c index 81a41914..ce9afc4d 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2242,6 +2242,10 @@ static inline void disk_advance( block_off_t run_rem = 1; const unsigned char* ptr = (const unsigned char*)block; + /* stop inspecting block flags at the end of the requested window */ + if (extent_rem > blockmax - pos) + extent_rem = blockmax - pos; + *has_file_ptr = has_file; *has_invalid_ptr = has_invalid; @@ -2264,14 +2268,16 @@ static inline void disk_advance( * block, and at least one disk has an invalid parity block (CHG, REP, REBUILD, * or DELETED). * - * The sweep takes O(B + R * D) time, where B is the number of blocks inspected, + * Counts only positions in [blockmin, blockmax). + * The sweep takes O(D * log(E) + B + R * D) time, where E is the number of + * extents per disk, B is the number of blocks inspected within the window, * R is the number of event positions processed (including initialization), and * D is the number of disks. Cached event boundaries avoid rescanning block runs. * * The caller must ensure that the disk list, extent trees, and block states - * remain unchanged during the count, as this function does not acquire locks. + * remain unchanged during the count, as no locks are held during the sweep. */ -uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmax) +uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmin, block_off_t blockmax) { tommy_tree_node* cursors[RAID_DATA_MAX]; block_off_t next_events[RAID_DATA_MAX]; @@ -2280,7 +2286,7 @@ uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmax) unsigned disk_count = 0; tommy_node* node; uint64_t total_unsynced = 0; - block_off_t pos = 0; + block_off_t pos = blockmin; size_t stride = block_sizeof(); unsigned d; @@ -2294,9 +2300,10 @@ uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmax) for (node = state->disklist; node != 0; node = node->next) { struct snapraid_disk* disk = node->data; - cursors[disk_count] = tommy_tree_head(&disk->fs_parity); + struct snapraid_extent* ext = fs_par2extent_find_next(disk, blockmin); + cursors[disk_count] = ext ? &ext->parity_node : 0; /* force an initial event so flags are set before their first use */ - next_events[disk_count] = 0; + next_events[disk_count] = blockmin; ++disk_count; } @@ -2508,6 +2515,8 @@ struct state_read_context { static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f, struct state_read_context* context) { block_off_t blockmax; + block_off_t inv_min; + block_off_t inv_max; uint64_t count_file; uint64_t count_hardlink; uint64_t count_symlink; @@ -2523,7 +2532,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S int has_hash; int has_prevhash; int has_info; - int has_invalid_parity; char buffer[PATH_MAX]; ssize_t ret; tommy_array disk_mapping; @@ -2532,6 +2540,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S tommy_hashdyn bucket_hash; blockmax = 0; + inv_min = BLOCK_MAX; + inv_max = 0; count_file = 0; count_hardlink = 0; count_symlink = 0; @@ -2547,7 +2557,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S has_hash = 0; has_prevhash = 0; has_info = 0; - has_invalid_parity = 0; mapping_max = 0; memset(disk_file_count, 0, sizeof(disk_file_count)); tommy_array_init(&disk_mapping); @@ -2819,8 +2828,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } /* all file block states except BLK require a parity update */ - if (v_state != BLOCK_STATE_BLK) - has_invalid_parity = 1; + if (v_state != BLOCK_STATE_BLK) { + if (v_pos < inv_min) + inv_min = v_pos; + if (v_pos + v_count > inv_max) + inv_max = v_pos + v_count; + } v_file_pos = v_idx; ret = state_read_block_run(f, file, v_file_pos, v_count, v_state, discard_hash); @@ -2989,14 +3002,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } } } - - /* - * A synchronized content has no invalid block state, so its unsynced - * count is certainly zero. Scan the extent maps only when needed. - */ - if (has_invalid_parity) { - count_unsynced = fs_count_unsynced(state, blockmax); - } } else if (c == 'h') { /* hole */ block_off_t v_pos; @@ -3049,7 +3054,10 @@ static void state_read_content(struct snapraid_state* state, const char* path, S switch (c) { case 'o' : /* if it's a run of deleted blocks */ - has_invalid_parity = 1; + if (v_pos < inv_min) + inv_min = v_pos; + if (v_pos + v_count > inv_max) + inv_max = v_pos + v_count; /* ensure the converted byte size fits in signed data_off_t without overflow */ if (v_count > INT64_MAX / (uint64_t)state->block_size) { @@ -4023,6 +4031,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_STOP */ } + /* + * Every invalid file or deleted run expands this window while parsing. + * Outside it all parity is valid, so no position can be unsynced. + * Count only after all records are loaded, independently of the info record. + * A synchronized content leaves the window empty and needs no sweep. + */ + if (inv_min < inv_max) { + count_unsynced = fs_count_unsynced(state, inv_min, inv_max); + } + context->blockmax = blockmax; context->count_map = mapping_max; context->count_file = count_file; diff --git a/cmdline/state.h b/cmdline/state.h index 441e1415..68ee7515 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -242,10 +242,10 @@ void state_read(struct snapraid_state* state); /** * Count the number of unsynced parity positions using an event sweep across extents. - * Counts positions in [0, blockmax). The caller must ensure that the disk list, + * Counts positions in [blockmin, blockmax). The caller must ensure that the disk list, * extent trees, and block states remain unchanged for the duration of the call. */ -uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmax); +uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmin, block_off_t blockmax); /** * Write the new state. From a39fb132f8e2c99b2cdf2e7ca9a72b15a8aa81e6 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 7 Oct 2026 09:09:39 +0200 Subject: [PATCH 29/51] Normalize comment style --- AGENTS.md | 2 +- cmdline/app.h | 10 ++--- cmdline/check.c | 16 +++----- cmdline/device.c | 22 +++++------ cmdline/elem.c | 2 +- cmdline/elem.h | 8 ++-- cmdline/main.c | 8 ++-- cmdline/mingwapp.c | 10 ++--- cmdline/murmur3.c | 4 +- cmdline/museair.c | 2 +- cmdline/parity.c | 4 +- cmdline/scan.c | 4 +- cmdline/scrub.c | 2 +- cmdline/selftest.c | 10 ++--- cmdline/snapraid.c | 94 +++++++++++++++++++++++----------------------- cmdline/state.c | 10 ++--- cmdline/state.h | 2 +- cmdline/str.c | 2 +- cmdline/str.h | 4 +- cmdline/support.c | 12 +++--- cmdline/sync.c | 8 ++-- cmdline/thermal.c | 2 +- cmdline/unixapp.c | 30 +++++++-------- cmdline/util.c | 12 +++--- os/mingw.c | 50 ++++++++++++------------ os/mingw.h | 12 +++--- os/portable.h | 8 ++-- os/unix.c | 24 ++++++------ os/unix.h | 2 +- 29 files changed, 186 insertions(+), 190 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c6e0753e..b0d4244e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,7 +105,7 @@ SnapRAID is designed to recover from an interruption, including `SIGKILL`, at an - **Format**: Enforce via `uncrustify -c uncrustify.cfg --no-backup *.c *.h` (or `uncrustify -c linux.cfg --no-backup *.c *.h` for `raid/` directory and subdirectories) - **Naming**: Snake_case for functions, UPPER_CASE for macros/constants - **Indentation**: Tabs for indentation, no alignment (existing codebase style) -- **Comments**: C-style `/** */` for multiline comments; C `/* first letter lowercase */` for single-line inline notes +- **Comments**: Use `/** ... */` for multiline top comments preceding declarations and `/* ... */` for multiline comments within code. Use `/* first letter lowercase */` for single-line inline notes. - **Critical Comments**: Always add a comment at non-obvious critical points, especially around data-integrity invariants, crash recovery, fallback behavior, concurrency, and fatal versus best-effort error handling. Explain why the logic is required, not merely what the code does. - **Headers**: All `.h` files have include guards (`#ifndef __NAME_H`) - **Preferences**: Use 0 instead of NULL and '\0' diff --git a/cmdline/app.h b/cmdline/app.h index f53a2d74..4c3ceda2 100644 --- a/cmdline/app.h +++ b/cmdline/app.h @@ -87,7 +87,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har #define SNAPSHOT_SCAN "scan" #define SNAPSHOT_RETIRING_SUFFIX ".retiring" -/* +/** * Snapshots context */ struct fssnapshot_struct { @@ -213,7 +213,7 @@ int fssnapshot_replace(struct fssnapshot_struct* fss, const char* old_name, cons */ int fssnapshot_path(const struct fssnapshot_struct* fss, const char* name, char* path, size_t path_size); -/* +/** * Log file. * * This stream if fully buffered. @@ -257,7 +257,7 @@ extern int exit_degraded; */ #define SMART_FLAGS 256 -/* +/** * Counts command, transport, or controller-level errors reported by the device. * These reflect failed I/O operations not directly caused by media defects * (for example interface, firmware, or power-related errors). @@ -266,7 +266,7 @@ extern int exit_degraded; */ #define SMART_ERROR_PROTOCOL 257 -/* +/** * Counts media-level errors where data could not be reliably read or written. * These indicate actual storage surface or flash failures and may imply data loss. * This counter is cumulative and never resets to zero, even if the underlying @@ -387,7 +387,7 @@ struct devinfo_struct { char name[PATH_MAX]; /**< Name of the disk combined with the split index if any */ char mount[PATH_MAX]; /**< Mount point or other contained directory. */ char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Attributes to ignore */ char file[PATH_MAX]; /**< File device. */ #ifdef _WIN32 diff --git a/cmdline/check.c b/cmdline/check.c index 1a074244..5dcbed5f 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -14,7 +14,7 @@ #include "raid/raid.h" #include "raid/combo.h" -/* +/** * SnapRAID check/fix model * * Check verifies the data described by the content file against the files @@ -841,11 +841,9 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns * distinction disappears. */ - /************************************************************************/ - /* First strategy: parity represents CURRENT data. */ - /************************************************************************/ - /* + * First strategy: parity represents CURRENT data. + * * Under the CURRENT hypothesis: * * BLK hash CURRENT, parity normally synchronized @@ -1129,11 +1127,9 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns else log_tag("recover_sync:%" PRIu64 ":%u: Failed with %d attempts\n", pos, n, ret); - /************************************************************************/ - /* Second strategy: parity still represents OLD data. */ - /************************************************************************/ - /* + * Second strategy: parity still represents OLD data. + * * The CURRENT attempt failed, so retry under the opposite history * hypothesis: physical parity still represents the state before the * interrupted sync. @@ -1964,7 +1960,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna */ valid_parity = 1; - /* If the parity is used by at least one file */ + /* if the parity is used by at least one file */ used_parity = 0; /* keep track of the number of failed blocks */ diff --git a/cmdline/device.c b/cmdline/device.c index 274e5c77..759eb649 100644 --- a/cmdline/device.c +++ b/cmdline/device.c @@ -43,12 +43,12 @@ void device_name_set(devinfo_t* dev, const char* name, int index) */ #define SMART_MEASURES 256 -/* +/** * Divider for SMART attribute 5 */ static unsigned SMART_5_STEP = 1; -/* +/** * Failure rate for 30 days, for a disk * with SMART attribute 5 at a given value. */ @@ -107,12 +107,12 @@ static double SMART_5_R[SMART_MEASURES] = { 0.4249, }; -/* +/** * Divider for SMART attribute 187 */ static unsigned SMART_187_STEP = 1; -/* +/** * Failure rate for 30 days, for a disk * with SMART attribute 187 at a given value. */ @@ -171,12 +171,12 @@ static double SMART_187_R[SMART_MEASURES] = { 3.4520, }; -/* +/** * Divider for SMART attribute 188 */ static unsigned SMART_188_STEP = 1; -/* +/** * Failure rate for 30 days, for a disk * with SMART attribute 188 at a given value. */ @@ -235,12 +235,12 @@ static double SMART_188_R[SMART_MEASURES] = { 1.3731, }; -/* +/** * Divider for SMART attribute 197 */ static unsigned SMART_197_STEP = 1; -/* +/** * Failure rate for 30 days, for a disk * with SMART attribute 197 at a given value. */ @@ -299,12 +299,12 @@ static double SMART_197_R[SMART_MEASURES] = { 4.1131, }; -/* +/** * Divider for SMART attribute 198 */ static unsigned SMART_198_STEP = 1; -/* +/** * Failure rate for 30 days, for a disk * with SMART attribute 198 at a given value. */ @@ -423,7 +423,7 @@ static double smart_afr(devinfo_t* devinfo) } if ( - /** + /* * Don't check Command_Timeout (188) for Seagate disks. * * It's reported by users that for Archive SMR (Shingled Magnetic Recording) diff --git a/cmdline/elem.c b/cmdline/elem.c index 3ec737f7..92993e2e 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -291,7 +291,7 @@ static int filter_element(tommy_list* filterlist, struct snapraid_filter** reaso { tommy_node* i; - /** + /* * The rule is that the latest filter defines the behaviour of paths * that doesn't match any filter. * diff --git a/cmdline/elem.h b/cmdline/elem.h index b9a638ae..118544ef 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -16,7 +16,7 @@ /****************************************************************************/ /* snapraid */ -/* +/** * Maximum number of block positions in a data/parity disk span. * * With the default 256 KiB block size, this limits the maximum supported @@ -415,7 +415,7 @@ struct snapraid_extra { char name[PATH_MAX]; /**< Name of the disk. */ char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */ char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */ char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */ char fstype[FSINFO_MAX]; /**< Filesystem type */ @@ -454,7 +454,7 @@ struct snapraid_disk { struct fssnapshot_struct fss; char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */ char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */ char fstype[FSINFO_MAX]; /**< Filesystem type */ @@ -597,7 +597,7 @@ struct snapraid_parity { struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */ unsigned split_mac; /**< Number of parity splits. */ char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */ block_off_t total_blocks; /**< Number of total blocks. */ block_off_t free_blocks; /**< Number of free blocks at the last sync. */ diff --git a/cmdline/main.c b/cmdline/main.c index 6d38d0bf..0acd58b1 100644 --- a/cmdline/main.c +++ b/cmdline/main.c @@ -63,7 +63,7 @@ static int needs_quote(const WCHAR* arg) cmd[pos++] = (c); \ } while (0) -/* +/** * Append one argument using the Microsoft C runtime command-line * quoting rules. Use this when building a normal CreateProcessW() * command line that the target process will parse back into argv[]. @@ -146,10 +146,10 @@ static char* argutf8(const WCHAR* arg) return utf8_arg; } -/* Global variable to store child process handle */ +/* global variable to store child process handle */ static void* volatile child_process = NULL; -/* +/** * Safe atomic access helpers for child_process. * * The console_handler runs concurrently on an OS-managed thread created by @@ -169,7 +169,7 @@ static void child_process_write(HANDLE handle) InterlockedExchangePointer(&child_process, handle); } -/* Console control handler - forwards Ctrl+C, Ctrl+Break to child */ +/* console control handler - forwards Ctrl+C, Ctrl+Break to child */ static BOOL WINAPI console_handler(DWORD ctrl_type) { /* if no child, default behavior */ diff --git a/cmdline/mingwapp.c b/cmdline/mingwapp.c index 992eb409..23c28d10 100644 --- a/cmdline/mingwapp.c +++ b/cmdline/mingwapp.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifdef __MINGW32__ /* Only for MingW */ +#ifdef __MINGW32__ /* only for MingW */ #include "support.h" #include "raid/cpu.h" @@ -82,7 +82,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) { char* slash = strrchr(dir, '/'); - /** + /* * Cut the file name, but leave the last slash. * * This is done because a MSDN comment about using of UNC paths. @@ -135,7 +135,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har if (has_persistent_inode) { *has_persistent_inode = 0; - /** + /* * Reuse stored file IDs across scans only for filesystems known to * provide stable and unique IDs. An unknown or unavailable filesystem * must be treated as volatile: a false negative only loses inode-based @@ -250,7 +250,7 @@ static int base64_encode(const unsigned char* in, size_t in_len, char* out, size return 0; } -/* +/** * PowerShell helper * * Runs a PowerShell one-liner via os_spawn() and captures the first line @@ -2411,7 +2411,7 @@ void machineinfo(struct machineinfo_struct* info) info->memory_page_size = si.dwPageSize; info->cpu_cores_logical = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); - /* Query processor topology and cache hierarchy via GetLogicalProcessorInformationEx */ + /* query processor topology and cache hierarchy via GetLogicalProcessorInformationEx */ DWORD len = 0; GetLogicalProcessorInformationEx(RelationAll, 0, &len); if (GetLastError() == ERROR_INSUFFICIENT_BUFFER && len > 0) { diff --git a/cmdline/murmur3.c b/cmdline/murmur3.c index 51cc8b09..8f912a7b 100644 --- a/cmdline/murmur3.c +++ b/cmdline/murmur3.c @@ -14,7 +14,7 @@ // MurmurHash3 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. -/* Finalization mix - force all bits of a hash block to avalanche */ +/* finalization mix - force all bits of a hash block to avalanche */ static inline uint32_t fmix32(uint32_t h) { h ^= h >> 16; @@ -25,7 +25,7 @@ static inline uint32_t fmix32(uint32_t h) return h; } -/* +/** * Non-static variables are intentionally used instead of static const * because keeping these constants in CPU registers yields measurably * higher throughput than using 32-bit immediate constants in imul. diff --git a/cmdline/museair.c b/cmdline/museair.c index 1eaad941..8bbf8e92 100644 --- a/cmdline/museair.c +++ b/cmdline/museair.c @@ -20,7 +20,7 @@ * of this license, visit: https://creativecommons.org/publicdomain/zero/1.0/ */ -/* +/** * Analysis of the BFast variant and blinding multiplication * * Many fast non-cryptographic hashes (wyhash, rapidhash) use "folding diff --git a/cmdline/parity.c b/cmdline/parity.c index c572eb36..64643679 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -220,7 +220,7 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p /* LCOV_EXCL_STOP */ } - /** + /* * If the parity size is not yet set, set it now. * This happens when adding a new parity level or split that has * no persisted size yet. @@ -884,7 +884,7 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par /* LCOV_EXCL_STOP */ } - /** + /* * If the parity size is not yet set, set it now. * This happens when adding a new parity level or split that has * no persisted size yet. diff --git a/cmdline/scan.c b/cmdline/scan.c index 4eb0e4e3..a2376b56 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -1153,7 +1153,7 @@ static void scan_file_discover(struct snapraid_scan* scan, const char* sub, stru disc = discovery_alloc(sub, st); - /* Hardlink collision detection */ + /* hardlink collision detection */ if (disc->inode != INODE_INVALID) { struct snapraid_discovery* rep = tommy_hashdyn_search(&scan->file_discovery_inodeset, discovery_inode_compare_to_arg, &disc->inode, file_inode_hash(disc->inode)); if (!rep) { @@ -1161,7 +1161,7 @@ static void scan_file_discover(struct snapraid_scan* scan, const char* sub, stru disc->flag |= DISCOVERY_IS_CANONICAL; tommy_hashdyn_insert(&scan->file_discovery_inodeset, &disc->nodeset, disc, file_inode_hash(disc->inode)); } else { - /* Inode collision detected! */ + /* inode collision detected! */ #if HAVE_LSTAT_SYNC if (disk->has_volatile_hardlinks && !(rep->flag & DISCOVERY_IS_SCAN_SYNCED)) { char path_next[PATH_MAX]; diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 481e6d1d..2a2ae827 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -11,7 +11,7 @@ #include "io.h" #include "raid/raid.h" -/* +/** * SnapRAID scrub model * * Scrub periodically verifies data and parity without modifying either of diff --git a/cmdline/selftest.c b/cmdline/selftest.c index 602f80e3..e72d35dc 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -1979,13 +1979,13 @@ static void test_smart_ignore(void) memset(&state, 0, sizeof(state)); memset(&devinfo, 0, sizeof(devinfo)); - /* Initialize devinfo attributes with mock data */ + /* initialize devinfo attributes with mock data */ for (i = 0; i < SMART_COUNT; ++i) { devinfo.smart[i].raw = 100; devinfo.smart[i].norm = 100; snprintf(devinfo.smart[i].name, sizeof(devinfo.smart[i].name), "Attr_%d", i); } - /* Special name for test */ + /* special name for test */ pathcpy(devinfo.smart[5].name, sizeof(devinfo.smart[5].name), "Reallocated_Sector_Ct"); pathcpy(devinfo.smart[197].name, sizeof(devinfo.smart[197].name), "Current_Pending_Sector"); @@ -2007,7 +2007,7 @@ static void test_smart_ignore(void) state_smart_ignore(&state, &devinfo); - /* Check that ignored attributes are set to SMART_UNASSIGNED */ + /* check that ignored attributes are set to SMART_UNASSIGNED */ if (devinfo.smart[5].raw != SMART_UNASSIGNED || devinfo.smart[5].norm != SMART_UNASSIGNED) { log_fatal(EINTERNAL, "test_smart_ignore: numerical global ignore failed\n"); exit(EXIT_FAILURE); @@ -2025,7 +2025,7 @@ static void test_smart_ignore(void) exit(EXIT_FAILURE); } - /* Check that non-ignored attributes are still present */ + /* check that non-ignored attributes are still present */ if (devinfo.smart[6].raw != 100) { log_fatal(EINTERNAL, "test_smart_ignore: unmodified attribute was cleared\n"); exit(EXIT_FAILURE); @@ -2345,7 +2345,7 @@ static void test_unsynced_sweep(void) block_state_set(file_block(file4, 2), BLOCK_STATE_CHG); fs_allocate(disk1, 30, file4, 0, 4); - /** + /* * Positions 30..33 are all unsynced: REP, REP, CHG, REBUILD respectively. * At 32 and 33 the invalid block changes disks at the same event boundary; * count each position once using the refreshed flags from both disks. diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 2d806fb9..c3c2f258 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -311,152 +311,152 @@ static struct option long_options[] = { { "gui-threshold-removes", 1, 0, OPT_GUI_THRESHOLD_REMOVES }, /* undocumented GUI, abort sync if too many removed files */ { "gui-threshold-updates", 1, 0, OPT_GUI_THRESHOLD_UPDATES }, /* undocumented GUI, abort sync if too many updated files */ - /* The following are test specific options, DO NOT USE! */ + /* the following are test specific options, DO NOT USE! */ - /* After syncing, do not write the new content file */ + /* after syncing, do not write the new content file */ { "test-kill-after-sync", 0, 0, OPT_TEST_KILL_AFTER_SYNC }, - /* Do not execute syncing after resizing parity files, before the normal post-resize content write */ + /* do not execute syncing after resizing parity files, before the normal post-resize content write */ { "test-kill-after-resize", 0, 0, OPT_TEST_KILL_AFTER_RESIZE }, - /* Do not execute syncing after writing the new initial content file */ + /* do not execute syncing after writing the new initial content file */ { "test-kill-before-sync", 0, 0, OPT_TEST_KILL_BEFORE_SYNC }, - /* Exit with failure if after check/fix there ARE NOT unrecoverable errors. */ + /* exit with failure if after check/fix there ARE NOT unrecoverable errors. */ { "test-expect-unrecoverable", 0, 0, OPT_TEST_EXPECT_UNRECOVERABLE }, - /* Exit with failure if after check/fix there ARE NOT recoverable errors. */ + /* exit with failure if after check/fix there ARE NOT recoverable errors. */ { "test-expect-recoverable", 0, 0, OPT_TEST_EXPECT_RECOVERABLE }, - /* Skip the initial self test */ + /* skip the initial self test */ { "test-skip-self", 0, 0, OPT_TEST_SKIP_SELF }, - /* Skip the initial sign check when reading the content file */ + /* skip the initial sign check when reading the content file */ { "test-skip-sign", 0, 0, OPT_TEST_SKIP_SIGN }, - /* Skip the fallocate() when growing the parity files */ + /* skip the fallocate() when growing the parity files */ { "test-skip-fallocate", 0, 0, OPT_TEST_SKIP_FALLOCATE }, - /* Skip the device check */ + /* skip the device check */ { "test-skip-device", 0, 0, OPT_TEST_SKIP_DEVICE }, - /* Force Murmur3 hash */ + /* force Murmur3 hash */ { "test-force-murmur3", 0, 0, OPT_TEST_FORCE_MURMUR3 }, - /* Force Spooky2 hash */ + /* force Spooky2 hash */ { "test-force-spooky2", 0, 0, OPT_TEST_FORCE_SPOOKY2 }, - /* Skip the use of lock file */ + /* skip the use of lock file */ { "test-skip-lock", 0, 0, OPT_TEST_SKIP_LOCK }, - /* Force a sort order for files */ + /* force a sort order for files */ { "test-force-order-inode", 0, 0, OPT_TEST_FORCE_ORDER_INODE }, { "test-force-order-alpha", 0, 0, OPT_TEST_FORCE_ORDER_ALPHA }, { "test-force-order-dir", 0, 0, OPT_TEST_FORCE_ORDER_DIR }, - /* Force scrub of the specified number of blocks */ + /* force scrub of the specified number of blocks */ { "test-force-scrub-at", 1, 0, OPT_TEST_FORCE_SCRUB_AT }, - /* Force scrub of all the even blocks. This is really for testing, don't try it */ + /* force scrub of all the even blocks. This is really for testing, don't try it */ { "test-force-scrub-even", 0, 0, OPT_TEST_FORCE_SCRUB_EVEN }, - /* Force write of the content file even if no modification is done */ + /* force write of the content file even if no modification is done */ { "test-force-content-write", 0, 0, OPT_TEST_FORCE_CONTENT_WRITE }, - /* Relax the checks done at the content file */ + /* relax the checks done at the content file */ { "test-skip-content-check", 0, 0, OPT_TEST_SKIP_CONTENT_CHECK }, - /* Skip the parity access */ + /* skip the parity access */ { "test-skip-parity-access", 0, 0, OPT_TEST_SKIP_PARITY_ACCESS }, - /* Exit generic failure */ + /* exit generic failure */ { "test-expect-failure", 0, 0, OPT_TEST_EXPECT_FAILURE }, - /* Exit generic need sync */ + /* exit generic need sync */ { "test-expect-need-sync", 0, 0, OPT_TEST_EXPECT_NEED_SYNC }, #if HAVE_CHECKER - /* Run some command after loading the state and before the command */ + /* run some command after loading the state and before the command */ { "test-run", 1, 0, OPT_TEST_RUN }, #endif - /* Use the FindFirst/Next approach in Windows to list files */ + /* use the FindFirst/Next approach in Windows to list files */ { "test-force-scan-winfind", 0, 0, OPT_TEST_FORCE_SCAN_WINFIND }, - /* Alternative import working by data */ + /* alternative import working by data */ { "test-import-content", 1, 0, OPT_TEST_IMPORT_CONTENT }, - /* Force immediate progress state update */ + /* force immediate progress state update */ { "test-force-progress", 0, 0, OPT_TEST_FORCE_PROGRESS }, - /* Skip the disk access */ + /* skip the disk access */ { "test-skip-disk-access", 0, 0, OPT_TEST_SKIP_DISK_ACCESS }, - /* Force autosave at the specified block */ + /* force autosave at the specified block */ { "test-force-autosave-at", 1, 0, OPT_TEST_FORCE_AUTOSAVE_AT }, - /* Fake device data */ + /* fake device data */ { "test-fake-device", 0, 0, OPT_TEST_FAKE_DEVICE }, - /* Fake UUID */ + /* fake UUID */ { "test-fake-uuid", 0, 0, OPT_TEST_FAKE_UUID }, - /* Match first UUID */ + /* match first UUID */ { "test-match-first-uuid", 0, 0, OPT_TEST_MATCH_FIRST_UUID }, - /* Force parity update even if all the data hash is already matching */ + /* force parity update even if all the data hash is already matching */ { "test-force-parity-update", 0, 0, OPT_TEST_FORCE_PARITY_UPDATE }, - /* Number of IO buffers */ + /* number of IO buffers */ { "test-io-cache", 1, 0, OPT_TEST_IO_CACHE }, - /* Print IO stats */ + /* print IO stats */ { "test-io-stats", 0, 0, OPT_TEST_IO_STATS }, /* now replaced by -A, --stats */ - /* Signal condition variable outside the mutex */ + /* signal condition variable outside the mutex */ { "test-cond-signal-outside", 0, 0, OPT_TEST_COND_SIGNAL_OUTSIDE }, - /* Set the io advise to none */ + /* set the io advise to none */ { "test-io-advise-none", 0, 0, OPT_TEST_IO_ADVISE_NONE }, - /* Set the io advise to sequential */ + /* set the io advise to sequential */ { "test-io-advise-sequential", 0, 0, OPT_TEST_IO_ADVISE_SEQUENTIAL }, - /* Set the io advise to flush */ + /* set the io advise to flush */ { "test-io-advise-flush", 0, 0, OPT_TEST_IO_ADVISE_FLUSH }, - /* Set the io advise to flush window */ + /* set the io advise to flush window */ { "test-io-advise-flush-window", 0, 0, OPT_TEST_IO_ADVISE_FLUSH_WINDOW }, - /* Set the io advise to discard */ + /* set the io advise to discard */ { "test-io-advise-discard", 0, 0, OPT_TEST_IO_ADVISE_DISCARD }, - /* Set the io advise to discard window */ + /* set the io advise to discard window */ { "test-io-advise-discard-window", 0, 0, OPT_TEST_IO_ADVISE_DISCARD_WINDOW }, - /* Set the io advise to direct */ + /* set the io advise to direct */ { "test-io-advise-direct", 0, 0, OPT_TEST_IO_ADVISE_DIRECT }, - /* Set an artificial parity limit */ + /* set an artificial parity limit */ { "test-parity-limit", 1, 0, OPT_TEST_PARITY_LIMIT }, - /* Skip content write */ + /* skip content write */ { "test-skip-content-write", 0, 0, OPT_TEST_SKIP_CONTENT_WRITE }, - /* Skip space holder file in parity disks */ + /* skip space holder file in parity disks */ { "test-skip-space-holder", 0, 0, OPT_TEST_SKIP_SPACE_HOLDER }, - /* Set the output format */ + /* set the output format */ { "test-fmt", 1, 0, OPT_TEST_FORMAT }, - /* Skip thread in disk scan */ + /* skip thread in disk scan */ { "test-skip-multi-scan", 0, 0, OPT_TEST_SKIP_MULTI_SCAN }, { 0, 0, 0, 0 } }; #endif -/* +/** * Free letters: gIjJkKMnPQruxXz * * The 's' letter is used in main.c diff --git a/cmdline/state.c b/cmdline/state.c index ce9afc4d..c7be0926 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -32,7 +32,7 @@ #define HAVE_MT_VERIFY 1 #endif -/* +/** * Start content-file writeback in bounded windows to avoid dirty-page * throttling; the subsequent fsync() remains responsible for durability. * @@ -4397,7 +4397,7 @@ static void* state_write_thread(void* arg) if (v_state != block_state_get(block)) break; - /* Resolve the mapping only when crossing an extent boundary. */ + /* resolve the mapping only when crossing an extent boundary. */ if (end == extent_end) { block_off_t next_count; block_off_t next_pos = fs_file2par_get_run(disk, file, end, &next_count); @@ -5702,7 +5702,7 @@ static void state_rename_content(struct snapraid_state* state) #endif } -/* +/** * Check if a modification time nanosecond value can be represented in the * content format. * @@ -5714,7 +5714,7 @@ static int state_write_mtime_nsec_valid(int mtime_nsec) return mtime_nsec == STAT_NSEC_INVALID || (mtime_nsec >= 0 && mtime_nsec < 1000000000); } -/* +/** * Validate invariants required by the persistent content format. * * state_write() must never serialize a state that state_read() would reject. @@ -5807,7 +5807,7 @@ static void state_write_check(struct snapraid_state* state) } } -/* +/** * Emit structured mapping tags for data disks and parity splits after * content files have been durably published. */ diff --git a/cmdline/state.h b/cmdline/state.h index 68ee7515..f9eb37ce 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -167,7 +167,7 @@ struct snapraid_state { tommy_list bucketlist; /**< Sorted list of info bucket. Derived from infoarr, but updated only state_read()/state_write() */ block_off_t bucketcount; /**< Sum of all the counts of the list */ - /* Thermal */ + /* thermal */ tommy_list thermallist; /**< List of all thermal state */ int thermal_stop_gathering; /**< If thermal data gathering is stopped */ int thermal_system_temperature; /**< System temperature. 0 if not available */ diff --git a/cmdline/str.c b/cmdline/str.c index aac50593..45dbb3c2 100644 --- a/cmdline/str.c +++ b/cmdline/str.c @@ -493,7 +493,7 @@ static inline int is_wild_escape(char c) #endif } -/* +/** * Match with the previous pattern character provided in prev. * * An initial '**' needs to know if it was preceded by '/' to distinguish diff --git a/cmdline/str.h b/cmdline/str.h index a11c8a0c..bbc4f072 100644 --- a/cmdline/str.h +++ b/cmdline/str.h @@ -97,7 +97,7 @@ char* strtrim(char* s); */ char* strlwr(char* s); -/* +/** * Find the first occurrence of 'needle' in 'haystack' only if it * appears as a separate word (space, number or string boundaries). * @@ -127,7 +127,7 @@ unsigned long long strtonll(const char* nptr, char** endptr, int base); /****************************************************************************/ /* match */ -/* +/** * Wild match function. * * If match_sub is !=0, it matches sub directory. Specifically it matches if the diff --git a/cmdline/support.c b/cmdline/support.c index bbb91d7d..13983ec6 100644 --- a/cmdline/support.c +++ b/cmdline/support.c @@ -133,7 +133,7 @@ unsigned char random_u8(void) return random_u64() & 0xFF; } -/* +/** * SplitMix64 * * - Zero-safe: Any 64-bit seed is valid (even 0). @@ -293,7 +293,7 @@ unsigned log_hardware_errors(void) return count; } -/* +/** * Note that in the following functions we always flush both * stdout and stderr, because we want to ensure that they mixes * well when redirected to files @@ -395,7 +395,7 @@ void vlog_fatal(int err, const char* format, va_list ap, const char* post) log_failed = 1; } - /* A failed tagged write must retain stderr as the only usable fatal diagnostic. */ + /* a failed tagged write must retain stderr as the only usable fatal diagnostic. */ if (!stdlog || log_failed || msg_fatal_stderr) { va_list ap_copy2; va_copy(ap_copy2, ap); @@ -1108,7 +1108,7 @@ void pathimport(char* dst, size_t size, const char* src) pathcpy(dst, size, src); #ifdef _WIN32 - /* Convert the Windows dir separator '\' to C '/' */ + /* convert the Windows dir separator '\' to C '/' */ while (*dst) { if (*dst == '\\') *dst = '/'; @@ -1122,7 +1122,7 @@ void pathexport(char* dst, size_t size, const char* src) pathcpy(dst, size, src); #ifdef _WIN32 - /* Convert C '/' to Windows dir separator '\' */ + /* convert C '/' to Windows dir separator '\' */ while (*dst) { if (*dst == '/') *dst = '\\'; @@ -1928,7 +1928,7 @@ int is_sep(char c) || c == (char)0xa0; /* french */ } -/* +/** * snumber() * * Matches a literal prefix (the second argument), then parses the following diff --git a/cmdline/sync.c b/cmdline/sync.c index c9b505c3..8d8770aa 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -11,7 +11,7 @@ #include "io.h" #include "raid/raid.h" -/* +/** * SnapRAID sync model * * Sync brings the stored parity and content metadata in line with the data @@ -1921,7 +1921,7 @@ bail_abort: return 0; } -/* +/** * Make a forced full parity rebuild persistent. * * REBUILD means that the current data and hash are known, but physical parity @@ -1959,7 +1959,7 @@ static void state_sync_mark_rebuild(struct snapraid_state* state) state->need_write = 1; } -/* +/** * Initialize persistent parity split sizes resolved while opening parity. * * parity_create() resolves PARITY_SIZE_INVALID in the parity handles using @@ -2208,7 +2208,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t /* after resizing parity files, refresh again the free info */ state_refresh(state); - /** + /* * Save the new state before the sync but after the hashing phase * * This allows to recover after an aborted sync, and at the same time diff --git a/cmdline/thermal.c b/cmdline/thermal.c index 821d9675..ff0c05d7 100644 --- a/cmdline/thermal.c +++ b/cmdline/thermal.c @@ -30,7 +30,7 @@ void thermal_free(void* void_thermal) free(thermal); } -/* +/** * Fit exponential heating model to data using least squares */ struct snapraid_thermal_params fit_thermal_model(const struct snapraid_thermal_point* points, int n_points, double t_ambient) diff --git a/cmdline/unixapp.c b/cmdline/unixapp.c index 20e52be8..78479c05 100644 --- a/cmdline/unixapp.c +++ b/cmdline/unixapp.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifndef __MINGW32__ /* Only for Unix */ +#ifndef __MINGW32__ /* only for Unix */ #include "app.h" #include "support.h" @@ -64,7 +64,7 @@ static inline unsigned statfs_type(const struct statfs* sfs) #if HAVE_LINUX_DEVICE static const char* const bcachefs_paths[] = { #ifdef BCACHEFS_PATH - /* Path configured at build time (e.g. on NixOS). */ + /* path configured at build time (e.g. on NixOS). */ BCACHEFS_PATH, #else "/usr/sbin/bcachefs", @@ -96,7 +96,7 @@ static const char* find_bcachefs(void) #if HAVE_LINUX_DEVICE static const char* const zfs_paths[] = { #ifdef ZFS_PATH - /* Path configured at build time (e.g. on NixOS). */ + /* path configured at build time (e.g. on NixOS). */ ZFS_PATH, #else "/usr/sbin/zfs", @@ -125,7 +125,7 @@ static const char* find_zfs(void) static const char* const zpool_paths[] = { #ifdef ZPOOL_PATH - /* Path configured at build time (e.g. on NixOS). */ + /* path configured at build time (e.g. on NixOS). */ ZPOOL_PATH, #else "/usr/sbin/zpool", @@ -157,7 +157,7 @@ static const char* find_zpool(void) static const char* smartctl_paths[] = { #ifdef SMARTCTL_PATH - /* Path configured at build time (e.g. on NixOS). */ + /* path configured at build time (e.g. on NixOS). */ SMARTCTL_PATH, #else /* Linux & BSD */ @@ -228,7 +228,7 @@ static ssize_t sysread(const char* path, char* buf, size_t buf_size) #endif #if HAVE_LINUX_DEVICE -/* +/** * sysattr_vpd_pg80 — parse raw binary VPD page 0x80 (Unit Serial Number) * from a sysfs vpd_pg80 file. */ @@ -291,7 +291,7 @@ static int sysattr_vpd_pg80(const char* path, char* dst, size_t dst_size) } #endif -/* +/** * sysread_vpd_pg83 — parse raw binary VPD page 0x83 (Device Identification) * from a sysfs vpd_pg83 file and extract the first NAA (Network Address * Authority) designator for the Logical Unit (ASSOCIATION == 0x00). @@ -418,7 +418,7 @@ static int sysattr_vpd_pg83(const char* path, char* dst, size_t dst_size) switch (naa) { case 0x1 : /* IEEE Extended */ - case 0x2 : /* Locally Assigned */ + case 0x2 : /* locally assigned */ case 0x3 : /* IEEE Registered */ case 0x5 : /* IEEE Registered */ expected = 8; @@ -503,7 +503,7 @@ static long long syslong(const char* path) return v; } -/* +/** * Parse Linux cpulist format (e.g. "0-3,5,8-11") into an array of CPU IDs. * * Returns the total number of CPUs in the list. Stores up to max_cpus entries. @@ -553,7 +553,7 @@ static unsigned parse_cpulist(const char* str, unsigned* cpus, unsigned max_cpus return count; } -/* +/** * Read online CPU IDs from /sys/devices/system/cpu/online. * * Falls back to consecutive 0..N-1 IDs if sysfs is unavailable. @@ -584,7 +584,7 @@ static unsigned read_online_cpus(unsigned* cpus, unsigned max_cpus) return count; } -/* +/** * Returns 1 if the specified cpu is the lowest-numbered logical CPU in its * physical core's SMT sibling set. */ @@ -2737,7 +2737,7 @@ static int fssnapshot_stat_fs(const struct fssnapshot_struct* fss, const char* n } #endif -/* +/** * Native ZFS snapshots use a SnapRAID-specific prefix to avoid * deleting or renaming unrelated user snapshots with generic * names such as @scan, @pending, or @stable. @@ -2772,7 +2772,7 @@ static int fssnapshot_stat_zfs(const struct fssnapshot_struct* fss, const char* return -1; } - /** + /* * When verifying if a ZFS snapshot has been successfully deleted, a direct * path lookup (stat, access, open) is unreliable. A directory listing * (readdir) of the .zfs/snapshot directory must be used instead. @@ -4530,7 +4530,7 @@ int devmap(void) #if HAVE_LINUX_DEVICE -/* List of possible ambient temperature labels */ +/* list of possible ambient temperature labels */ const char* AMBIENT_LABEL[] = { "systin", "auxtin", @@ -4976,7 +4976,7 @@ void machineinfo(struct machineinfo_struct* info) } } - /* Fallback if sysfs cpufreq is unavailable (e.g. Cloud VM) */ + /* fallback if sysfs cpufreq is unavailable (e.g. Cloud VM) */ if (info->cpu_clock[0] == 0) { #ifdef CONFIG_X86 raid_cpuid(0, 0, reg); diff --git a/cmdline/util.c b/cmdline/util.c index c6c2e1c4..8cfbde5b 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -433,7 +433,7 @@ static void crc32c_skip_init(uint32_t table[8][16], size_t size) unsigned i; unsigned j; - /* Collapse each fixed-size shift into eight nibble table lookups. */ + /* collapse each fixed-size shift into eight nibble table lookups. */ for (i = 0; i < 8; ++i) { for (j = 0; j < 16; ++j) table[i][j] = crc32c_shift((uint32_t)j << (4 * i), size); @@ -514,7 +514,7 @@ void crc32c_init(void) /****************************************************************************/ /* byte operations */ -/* +/** * Rotate left. * In x86/x64/ARM they are optimized with a single assembler instruction. */ @@ -528,7 +528,7 @@ static inline uint64_t util_rotl64(uint64_t x, unsigned r) return (x << (r & 63)) | (x >> ((-r) & 63)); } -/* +/** * Rotate right. * In x86/x64/ARM they are optimized with a single assembler instruction. */ @@ -610,7 +610,7 @@ static inline void util_write64(void* ptr, uint64_t v) memcpy(ptr, &v, sizeof(v)); } -/* +/** * 64*64 -> 128bit multiply function. * * @param A Address of 64-bit number. @@ -622,7 +622,7 @@ static inline void util_write64(void* ptr, uint64_t v) * Overwrites B contents with C's high 64 bits. */ #if defined(__x86_64__) || defined(_M_X64) -/* +/** * This is intentionally placed before the __SIZEOF_INT128__ implementation. * * This implementation happens to generate more efficient code. @@ -653,7 +653,7 @@ static inline void util_mum(uint64_t* A, uint64_t* B) *B = (uint64_t)(product >> 64); } #elif defined(__aarch64__) || defined(_M_ARM64) -/* +/** * This is intentionally placed after the __SIZEOF_INT128__ implementation. * * Although the code looks correct, there is a report of this diff --git a/os/mingw.c b/os/mingw.c index e457ea30..639f3073 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifdef __MINGW32__ /* Only for MingW */ +#ifdef __MINGW32__ /* only for MingW */ #include "os.h" @@ -14,9 +14,9 @@ /****************************************************************************/ /* global */ -/* Add missing Windows declaration */ +/* add missing Windows declaration */ -/* For SetThreadExecutionState */ +/* for SetThreadExecutionState */ #ifndef WIN32_ES_SYSTEM_REQUIRED #define WIN32_ES_SYSTEM_REQUIRED 0x00000001L #endif @@ -33,7 +33,7 @@ #define WIN32_ES_CONTINUOUS 0x80000000L #endif -/* For SetSearchPathMode */ +/* for SetSearchPathMode */ #ifndef BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE #define BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE 0x00000001 #endif @@ -41,7 +41,7 @@ #define BASE_SEARCH_PATH_PERMANENT 0x00008000 #endif -/* File Index */ +/* file Index */ #undef FILE_INVALID_FILE_ID #define FILE_INVALID_FILE_ID ((ULONGLONG)-1LL) #undef FILE_UNSUPPORTED_FILE_ID @@ -571,17 +571,17 @@ static BOOL GetReparseTagInfoByHandle(HANDLE hFile, FILE_ATTRIBUTE_TAG_INFO* lpF */ static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct windows_stat* st) { - /* Convert special attributes */ + /* convert special attributes */ if ((FileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { st->st_mode = S_IFBLK; st->st_desc = "device"; - } else if ((FileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* Offline */ + } else if ((FileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* offline */ st->st_mode = S_IFCHR; st->st_desc = "offline"; - } else if ((FileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* Temporary */ + } else if ((FileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* temporary */ st->st_mode = S_IFCHR; st->st_desc = "temporary"; - } else if ((FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* Reparse point */ + } else if ((FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* reparse point */ switch (ReparseTag) { /* if we don't have the ReparseTag information */ case 0 : @@ -622,7 +622,7 @@ static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct win st->st_desc = "reparse-point"; break; } - } else if ((FileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* System */ + } else if ((FileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* system */ if ((FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { st->st_mode = S_IFCHR; st->st_desc = "system-directory"; @@ -696,7 +696,7 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ /* GetFileInformationByHandle() ensures to return synced information */ st->st_sync = 1; - /** + /* * In ReFS the IDs are 128 bit, and the 64 bit interface may fail. * * From Microsoft "Application Compatibility with ReFS" @@ -799,13 +799,13 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s windows_time2stat(mtime, st); - /* No inode information available */ + /* no inode information available */ st->st_ino = INODE_INVALID; - /* No link information available */ + /* no link information available */ st->st_nlink = 0; - /* No device information available */ + /* no device information available */ st->st_dev = 0; /* directory listing doesn't ensure to return synced information */ @@ -2079,7 +2079,7 @@ int windows_symlink_directory(const char* existing, const char* file) return windows_symlink_flags(existing, file, SYMBOLIC_LINK_FLAG_DIRECTORY); } -/* Adds missing definitions in MingW winnt.h */ +/* adds missing definitions in MingW winnt.h */ #ifndef FSCTL_GET_REPARSE_POINT #define FSCTL_GET_REPARSE_POINT 0x000900a8 #endif @@ -2258,7 +2258,7 @@ ssize_t windows_read(int fd, void* buffer, size_t size) if (!ReadFile(h, buffer, size, &count, 0)) { DWORD error = GetLastError(); - /* A closed Windows pipe has the same read semantics as a POSIX pipe at EOF. */ + /* a closed Windows pipe has the same read semantics as a POSIX pipe at EOF. */ if (error == ERROR_BROKEN_PIPE) return 0; @@ -2982,7 +2982,7 @@ static int fixcat(WCHAR* cmd, int size, int pos, const WCHAR* arg) return pos; } -/* +/** * Append one argument using the Microsoft C runtime command-line * quoting rules. Use this when building a normal CreateProcessW() * command line that the target process will parse back into argv[]. @@ -3043,7 +3043,7 @@ static int argcat(WCHAR* cmd, int size, int pos, const WCHAR* arg) return pos; } -/* +/** * Append one argument for a batch script invoked through cmd.exe /c. * Unlike argcat(), this must preserve the argument boundary while * preventing cmd.exe command metacharacters from changing the command @@ -3369,10 +3369,10 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch if (!ret) create_error = GetLastError(); } else { - /* Drop to restricted service account */ + /* drop to restricted service account */ HANDLE h_token = NULL; - /* Validate that the requested user is actually a supported Service Account before attempting logon */ + /* validate that the requested user is actually a supported Service Account before attempting logon */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); @@ -3407,7 +3407,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch return -1; } - /* Create an environment block to ensure PATH is loaded */ + /* create an environment block to ensure PATH is loaded */ LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); @@ -4385,7 +4385,7 @@ OS_FILE* os_popen(const char** argv) fail_child: saved_errno = errno; close(stdout_fd); - /* Terminate and reap the child immediately so no process or handle is leaked on stream creation failure. */ + /* terminate and reap the child immediately so no process or handle is leaked on stream creation failure. */ pid_kill(pid); status = 0; os_wait(pid, &status, 0); @@ -4409,15 +4409,15 @@ int os_pclose(OS_FILE* stream) int close_ret; pid_t wait_ret; - /* Keep the owned resources locally so the OS_FILE container can be released immediately. */ + /* keep the owned resources locally so the OS_FILE container can be released immediately. */ free(stream); - /* Closing the stream also closes the CRT descriptor and the underlying stdout pipe handle. */ + /* closing the stream also closes the CRT descriptor and the underlying stdout pipe handle. */ close_ret = fclose(fp); if (close_ret != 0) saved_errno = errno; - /* Always reap the child even if closing the stream failed. */ + /* always reap the child even if closing the stream failed. */ wait_ret = os_wait(pid, &status, 0); if (wait_ret < 0 && saved_errno == 0) saved_errno = errno; diff --git a/os/mingw.h b/os/mingw.h index 13474359..3dcbd7ef 100644 --- a/os/mingw.h +++ b/os/mingw.h @@ -4,7 +4,7 @@ #ifndef __PORTABLE_MINGW_H #define __PORTABLE_MINGW_H -#ifdef __MINGW32__ /* Only for MingW */ +#ifdef __MINGW32__ /* only for MingW */ #include @@ -98,7 +98,7 @@ wchar_t* u8tou16_mayfail(wchar_t* conv_buf, size_t number_of_wchar, const char* #undef PATH_MAX #define PATH_MAX 1024 -/* Remap functions and types */ +/* remap functions and types */ #undef fopen #define fopen windows_fopen #define FOPEN_TEXT "t" @@ -167,11 +167,11 @@ wchar_t* u8tou16_mayfail(wchar_t* conv_buf, size_t number_of_wchar, const char* #undef sleep #define sleep windows_sleep /* 4==DIR, 5,6,7=free, 8==REG */ -#define S_IFLNK 0x5000 /* Symbolic link to file */ +#define S_IFLNK 0x5000 /* symbolic link to file */ #define S_ISLNK(m) (((m) & _S_IFMT) == S_IFLNK) -#define S_IFLNKDIR 0x6000 /* Symbolic link to directory */ +#define S_IFLNKDIR 0x6000 /* symbolic link to directory */ #define S_ISLNKDIR(m) (((m) & _S_IFMT) == S_IFLNKDIR) -#define S_IFJUN 0x7000 /* Junction */ +#define S_IFJUN 0x7000 /* junction */ #define S_ISJUN(m) (((m) & _S_IFMT) == S_IFJUN) #undef readlink #define readlink windows_readlink @@ -210,7 +210,7 @@ wchar_t* u8tou16_mayfail(wchar_t* conv_buf, size_t number_of_wchar, const char* */ #define STAT_NSEC_INVALID -1 -/* We have nano second support */ +/* we have nano second support */ #define STAT_NSEC(st) ((int)(st)->st_mtimensec) static inline int WEXITSTATUS(DWORD status) diff --git a/os/portable.h b/os/portable.h index 6336600e..516c24cf 100644 --- a/os/portable.h +++ b/os/portable.h @@ -5,11 +5,11 @@ #define __PORTABLE_H #if HAVE_CONFIG_H -#include "config.h" /* Use " to include first in the same directory of this file */ +#include "config.h" /* use " to include first in the same directory of this file */ #endif /***************************************************************************/ -/* Config */ +/* config */ #ifdef __MINGW32__ /** @@ -163,7 +163,7 @@ * Includes some standard headers. */ #include -#include /* On many systems (e.g., Darwin), `stdio.h' is a prerequisite. */ +#include /* on many systems (e.g., Darwin), `stdio.h' is a prerequisite. */ #include #include #include @@ -299,7 +299,7 @@ #include #endif -/* +/** * Flags for open() */ #ifndef O_PATH diff --git a/os/unix.c b/os/unix.c index 41215329..9c4bcbf0 100644 --- a/os/unix.c +++ b/os/unix.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifndef __MINGW32__ /* Only for Unix */ +#ifndef __MINGW32__ /* only for Unix */ #include "os.h" @@ -353,7 +353,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical) /****************************************************************************/ /* exec */ -/* +/** * Scrubbed environment * Only provide the bare essentials. */ @@ -369,7 +369,7 @@ static char* const envp_scrubbed[] = { NULL }; -/* +/** * Enforce a strict 128-byte shebang limit to match the actual Linux kernel behavior * and prevent security risks from truncation. * @@ -560,7 +560,7 @@ static int verify_shebang_interpreter(int fd, const char* script_path) return 0; } -/* +/** * Securely verify and open an executable file. * * Performs a series of security checks on the file at @exec_path before @@ -940,7 +940,7 @@ static int pid_kill(pid_t pid) return -1; } - /* Send SIGKILL signal to the negative PID to target the entire Process Group */ + /* send SIGKILL signal to the negative PID to target the entire Process Group */ int ret = kill(-pid, SIGKILL); /* @@ -1219,7 +1219,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou os_user_drop_privileges(&user); #if defined(__linux__) && defined(PR_SET_PDEATHSIG) - /* Send SIGKILL to child if parent daemon dies unexpectedly */ + /* send SIGKILL to child if parent daemon dies unexpectedly */ prctl(PR_SET_PDEATHSIG, SIGKILL); /* exit if parent died before prctl to avoid becoming orphaned */ if (getppid() != ppid) { @@ -1432,13 +1432,13 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t setpgid(0, 0); if (pipe_fds[1] != -1) - close(pipe_fds[1]); /* Close unused write end */ + close(pipe_fds[1]); /* close unused write end */ /* drop privileges first (if configured) */ os_user_drop_privileges(&user); #if defined(__linux__) && defined(PR_SET_PDEATHSIG) - /* Send SIGKILL to child if parent daemon dies unexpectedly */ + /* send SIGKILL to child if parent daemon dies unexpectedly */ prctl(PR_SET_PDEATHSIG, SIGKILL); /* exit if parent died before prctl to avoid becoming orphaned */ if (getppid() != ppid) { @@ -1461,7 +1461,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t _exit(126); } - /* Redirect STDOUT and STDERR to /dev/null */ + /* redirect STDOUT and STDERR to /dev/null */ if (dup2(null_fd, STDOUT_FILENO) == -1 || dup2(null_fd, STDERR_FILENO) == -1) _exit(126); @@ -1657,7 +1657,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char os_user_drop_privileges(&user); #if defined(__linux__) && defined(PR_SET_PDEATHSIG) - /* Send SIGKILL to child if parent daemon dies unexpectedly */ + /* send SIGKILL to child if parent daemon dies unexpectedly */ prctl(PR_SET_PDEATHSIG, SIGKILL); /* exit if parent died before prctl to avoid becoming orphaned */ if (getppid() != ppid) { @@ -2164,7 +2164,7 @@ void os_privileges_release(void) } } - /* Drop UID last. */ + /* drop UID last. */ if (uid_dropped) { if (syscall(SYS_setresuid, (uid_t)-1, (uid_t)unpriv_uid, (uid_t)-1) != 0) { if (errno == EPERM) { @@ -2198,7 +2198,7 @@ void os_privileges_release(void) } } - /* Drop UID last. */ + /* drop UID last. */ if (uid_dropped) { if (seteuid(unpriv_uid) != 0) { if (errno == EPERM) { diff --git a/os/unix.h b/os/unix.h index 91775d54..5ac326aa 100644 --- a/os/unix.h +++ b/os/unix.h @@ -67,7 +67,7 @@ static inline int close_range_impl(unsigned int first, unsigned int last, unsign */ #define STAT_NSEC_INVALID -1 -/* Check if we have nanoseconds support */ +/* check if we have nanoseconds support */ #if HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC #define STAT_NSEC(st) ((int)(st)->st_mtim.tv_nsec) /* Linux */ #elif HAVE_STRUCT_STAT_ST_MTIMENSEC From 88c54f041e730517b0aad9063bcce32002b8f778 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 7 Oct 2026 18:44:12 +0200 Subject: [PATCH 30/51] Update to TommyDS v4.0 --- cmdline/elem.c | 8 +- cmdline/memory.c | 17 + cmdline/memory.h | 6 + cmdline/museair.c | 2 +- cmdline/selftest.c | 137 ++++---- cmdline/util.c | 119 +------ tommyds/tommy.c | 3 +- tommyds/tommyarray.c | 95 +++++- tommyds/tommyarray.h | 199 ++++++++++-- tommyds/tommyarrayblkof.c | 141 ++++++-- tommyds/tommyarrayblkof.h | 219 +++++++++++-- tommyds/tommychain.h | 101 ++++-- tommyds/tommyhash.c | 318 ++++++++---------- tommyds/tommyhash.h | 274 ++++++++++++++-- tommyds/tommyhashdyn.c | 397 ++++++++++++++++------- tommyds/tommyhashdyn.h | 367 +++++++++++++++++---- tommyds/tommylist.c | 91 +++++- tommyds/tommylist.h | 507 +++++++++++++++++++++++++---- tommyds/tommytree.c | 281 +++++++++------- tommyds/tommytree.h | 659 +++++++++++++++++++++++++++----------- tommyds/tommytypes.h | 186 +++++++---- 21 files changed, 3032 insertions(+), 1095 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 92993e2e..db02a55c 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -1604,8 +1604,8 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap extent = extent_alloc(parity_pos, file, file_pos, count); /* insert the extent in the trees */ - parity_extent = tommy_tree_insert(&disk->fs_parity, &extent->parity_node, extent); - file_extent = tommy_tree_insert(&disk->fs_file, &extent->file_node, extent); + parity_extent = tommy_tree_insert_unique(&disk->fs_parity, &extent->parity_node, extent); + file_extent = tommy_tree_insert_unique(&disk->fs_file, &extent->file_node, extent); if (parity_extent != extent || file_extent != extent) { /* LCOV_EXCL_START */ @@ -1689,8 +1689,8 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) second_extent = extent_alloc(extent->parity_pos + first_count + 1, extent->file, extent->file_pos + first_count + 1, second_count); /* insert the extent in the trees */ - parity_extent = tommy_tree_insert(&disk->fs_parity, &second_extent->parity_node, second_extent); - file_extent = tommy_tree_insert(&disk->fs_file, &second_extent->file_node, second_extent); + parity_extent = tommy_tree_insert_unique(&disk->fs_parity, &second_extent->parity_node, second_extent); + file_extent = tommy_tree_insert_unique(&disk->fs_file, &second_extent->file_node, second_extent); if (parity_extent != second_extent || file_extent != second_extent) { /* LCOV_EXCL_START */ diff --git a/cmdline/memory.c b/cmdline/memory.c index f2c395a3..ae600ea6 100644 --- a/cmdline/memory.c +++ b/cmdline/memory.c @@ -119,6 +119,23 @@ void* malloc_nofail(size_t size) return ptr; } +void* realloc_nofail(void* ptr, size_t size) +{ + /* keep zero-size requests allocated, as in malloc_nofail() */ + void* new_ptr = realloc(ptr, size ? size : 1); + + if (!new_ptr) { + /* LCOV_EXCL_START */ + malloc_fail(size); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + malloc_counter_inc(size); + + return new_ptr; +} + void* nalloc_nofail(size_t count, size_t size) { if (size != 0 && count > SIZE_MAX / size) { diff --git a/cmdline/memory.h b/cmdline/memory.h index 0d05f376..ef3e187a 100644 --- a/cmdline/memory.h +++ b/cmdline/memory.h @@ -23,6 +23,12 @@ void memory_single_thread(int single_thread); */ void* malloc_nofail(size_t size); +/** + * Safe realloc. + * If no memory is available, it aborts. + */ +void* realloc_nofail(void* ptr, size_t size); + /** * Safe malloc with calloc calling arguments. * If no memory is available, it aborts. diff --git a/cmdline/museair.c b/cmdline/museair.c index 8bbf8e92..2f765e9e 100644 --- a/cmdline/museair.c +++ b/cmdline/museair.c @@ -150,7 +150,7 @@ static const uint64_t MUSEAIR_CONSTANT[13] = { do { \ l = a; \ h = b; \ - util_mum(&l, &h); \ + tommy_mum(&l, &h); \ } while (0) #define likely(a) tommy_likely(a) diff --git a/cmdline/selftest.c b/cmdline/selftest.c index e72d35dc..cf6d5ead 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -51,25 +51,25 @@ struct hash_test_vector { * Test vectors for tommy_hash32 */ static struct hash32_test_vector TEST_HASH32[] = { - { "", 0, 0x8614384c, 0xa766795d }, - { "a", 1, 0x12c16c36, 0xa766795d }, - { "abc", 3, 0xc58e8af5, 0xa766795d }, - { "message digest", 14, 0x006b32f1, 0xa766795d }, - { "abcdefghijklmnopqrstuvwxyz", 26, 0x7e6fcfe0, 0xa766795d }, - { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0x8604adf8, 0xa766795d }, - { "The quick brown fox jumps over the lazy dog", 43, 0xdeba3d3a, 0xa766795d }, - { "\x00", 1, 0x4a7d1c33, 0xa766795d }, - { "\x16\x27", 2, 0x8b50899b, 0xa766795d }, - { "\xe2\x56\xb4", 3, 0x60406493, 0xa766795d }, - { "\xc9\x4d\x9c\xda", 4, 0xa049144a, 0xa766795d }, - { "\x79\xf1\x29\x69\x5d", 5, 0x4da2c2f1, 0xa766795d }, - { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x59de30cf, 0xa766795d }, - { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x219e149c, 0xa766795d }, - { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x25067520, 0xa766795d }, - { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0xa1f368d8, 0xa766795d }, - { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0x805fc63d, 0xa766795d }, - { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0x7f75dd0f, 0xa766795d }, - { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0xb9154382, 0xa766795d }, + { "", 0, 0xaffe48d7, 0xa766795d }, + { "a", 1, 0x2ae4a38d, 0xa766795d }, + { "abc", 3, 0x0aefd0ef, 0xa766795d }, + { "message digest", 14, 0x828c4a8a, 0xa766795d }, + { "abcdefghijklmnopqrstuvwxyz", 26, 0xc8072ea9, 0xa766795d }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0xe8e24812, 0xa766795d }, + { "The quick brown fox jumps over the lazy dog", 43, 0xae34960f, 0xa766795d }, + { "\x00", 1, 0xac5373be, 0xa766795d }, + { "\x16\x27", 2, 0x321c4629, 0xa766795d }, + { "\xe2\x56\xb4", 3, 0x66b8fbd8, 0xa766795d }, + { "\xc9\x4d\x9c\xda", 4, 0xbc37eb40, 0xa766795d }, + { "\x79\xf1\x29\x69\x5d", 5, 0xb96d1628, 0xa766795d }, + { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x760cf131, 0xa766795d }, + { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0xd1d5a37a, 0xa766795d }, + { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x995d591f, 0xa766795d }, + { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0x8c7cfa48, 0xa766795d }, + { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0xaa8fa306, 0xa766795d }, + { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0x9c1d4af8, 0xa766795d }, + { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x1a2214ec, 0xa766795d }, { 0, 0, 0, 0 } }; @@ -77,29 +77,29 @@ static struct hash32_test_vector TEST_HASH32[] = { * Test vectors for tommy_strhash32 */ struct strhash32_test_vector TEST_STRHASH32[] = { - { "", 0x0af1416d, 0xa766795d }, - { "a", 0x68fa0f3f, 0xa766795d }, - { "abc", 0xfc68ffc5, 0xa766795d }, - { "message digest", 0x08477b63, 0xa766795d }, - { "abcdefghijklmnopqrstuvwxyz", 0x5b9c25e5, 0xa766795d }, - { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x1e530ce7, 0xa766795d }, - { "The quick brown fox jumps over the lazy dog", 0xaf93eefe, 0xa766795d }, - { "\xff", 0xfc88801b, 0xa766795d }, - { "\x16\x27", 0xcd7216db, 0xa766795d }, - { "\xe2\x56\xb4", 0x05f98d02, 0xa766795d }, - { "\xc9\x4d\x9c\xda", 0xf65206f8, 0xa766795d }, - { "\x79\xf1\x29\x69\x5d", 0x72bd6bda, 0xa766795d }, - { "\xff\x7e\xdf\x1e\x31\x1c", 0x57dfb9b4, 0xa766795d }, - { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 0x499ff634, 0xa766795d }, - { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 0xe896b7ce, 0xa766795d }, - { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 0xfe3939f0, 0xa766795d }, - { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 0x4351d482, 0xa766795d }, - { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\xff\xb7\xae", 0x88e92135, 0xa766795d }, - { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 0x01109c16, 0xa766795d }, - { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 0xbcb050dc, 0xa766795d }, - { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 0xbe5e1fd5, 0xa766795d }, - { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 0x70d8c97f, 0xa766795d }, - { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 0x957440a9, 0xa766795d }, + { "", 0xaffe48d7, 0xa766795d }, + { "a", 0x2ae4a38d, 0xa766795d }, + { "abc", 0x0aefd0ef, 0xa766795d }, + { "message digest", 0x828c4a8a, 0xa766795d }, + { "abcdefghijklmnopqrstuvwxyz", 0xc8072ea9, 0xa766795d }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xe8e24812, 0xa766795d }, + { "The quick brown fox jumps over the lazy dog", 0xae34960f, 0xa766795d }, + { "\xff", 0x9ab96198, 0xa766795d }, + { "\x16\x27", 0x321c4629, 0xa766795d }, + { "\xe2\x56\xb4", 0x66b8fbd8, 0xa766795d }, + { "\xc9\x4d\x9c\xda", 0xbc37eb40, 0xa766795d }, + { "\x79\xf1\x29\x69\x5d", 0xb96d1628, 0xa766795d }, + { "\xff\x7e\xdf\x1e\x31\x1c", 0xaf63217b, 0xa766795d }, + { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 0xd1d5a37a, 0xa766795d }, + { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 0x995d591f, 0xa766795d }, + { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 0x8c7cfa48, 0xa766795d }, + { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 0xaa8fa306, 0xa766795d }, + { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\xff\xb7\xae", 0x59437055, 0xa766795d }, + { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 0x1a2214ec, 0xa766795d }, + { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 0xb5d1be68, 0xa766795d }, + { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 0xb2475cc8, 0xa766795d }, + { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 0x3cf85197, 0xa766795d }, + { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 0x8ec35c09, 0xa766795d }, { 0, 0, 0 } }; @@ -107,25 +107,25 @@ struct strhash32_test_vector TEST_STRHASH32[] = { * Test vectors for tommy_hash64 */ static struct hash64_test_vector TEST_HASH64[] = { - { "", 0, 0x8614384cb5165fbfULL, 0x2f022773a766795dULL }, - { "a", 1, 0x1a2e0298a8e94a3dULL, 0x2f022773a766795dULL }, - { "abc", 3, 0x7555796b7a7d21ebULL, 0x2f022773a766795dULL }, - { "message digest", 14, 0x9411a57d04b92fb4ULL, 0x2f022773a766795dULL }, - { "abcdefghijklmnopqrstuvwxyz", 26, 0x3ca3f8d2b4e69832ULL, 0x2f022773a766795dULL }, - { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0x6dae542ba0015a4dULL, 0x2f022773a766795dULL }, - { "The quick brown fox jumps over the lazy dog", 43, 0xe06d8cbb3d2ea1a6ULL, 0x2f022773a766795dULL }, - { "\x00", 1, 0x201e664fb5f2c021ULL, 0x2f022773a766795dULL }, - { "\x16\x27", 2, 0xef42fa8032c4b775ULL, 0x2f022773a766795dULL }, - { "\xe2\x56\xb4", 3, 0x6e6c498a6688466cULL, 0x2f022773a766795dULL }, - { "\xc9\x4d\x9c\xda", 4, 0x5195005419905423ULL, 0x2f022773a766795dULL }, - { "\x79\xf1\x29\x69\x5d", 5, 0x221235b48afee7c1ULL, 0x2f022773a766795dULL }, - { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x1b1f18b9266f095bULL, 0x2f022773a766795dULL }, - { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x2cbafa8e741d49caULL, 0x2f022773a766795dULL }, - { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x4677f04c06e0758dULL, 0x2f022773a766795dULL }, - { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0x5afe09e8214e2163ULL, 0x2f022773a766795dULL }, - { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0x115b6276d209fab6ULL, 0x2f022773a766795dULL }, - { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0xd0636d2f01cf3a3eULL, 0x2f022773a766795dULL }, - { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x6d259f5fef74f93eULL, 0x2f022773a766795dULL }, + { "", 0, 0xf64d2d2164ca4e35ULL, 0x2f022773a766795dULL }, + { "a", 1, 0xcb0b7863fd912608ULL, 0x2f022773a766795dULL }, + { "abc", 3, 0x023adbd4412dfbc2ULL, 0x2f022773a766795dULL }, + { "message digest", 14, 0x8d16f620b58a030eULL, 0x2f022773a766795dULL }, + { "abcdefghijklmnopqrstuvwxyz", 26, 0xa615ea6b87f2b63fULL, 0x2f022773a766795dULL }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0xb9f145676ebdb4fcULL, 0x2f022773a766795dULL }, + { "The quick brown fox jumps over the lazy dog", 43, 0x99b05c9548b523dcULL, 0x2f022773a766795dULL }, + { "\x00", 1, 0x432204e0ea8617fbULL, 0x2f022773a766795dULL }, + { "\x16\x27", 2, 0x5118c58166c6ab3eULL, 0x2f022773a766795dULL }, + { "\xe2\x56\xb4", 3, 0x29484604a9976acbULL, 0x2f022773a766795dULL }, + { "\xc9\x4d\x9c\xda", 4, 0xf000ac44a9f98f4aULL, 0x2f022773a766795dULL }, + { "\x79\xf1\x29\x69\x5d", 5, 0x9603fc74ec52e5aaULL, 0x2f022773a766795dULL }, + { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0xd39f549b363c8e2bULL, 0x2f022773a766795dULL }, + { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x1be47d9f53a2a31dULL, 0x2f022773a766795dULL }, + { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0xe3fa24e31af8c4ebULL, 0x2f022773a766795dULL }, + { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0xef8c3dcec426a25dULL, 0x2f022773a766795dULL }, + { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0xed0be01a88bde7cbULL, 0x2f022773a766795dULL }, + { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0xf9b50d9db5dfa543ULL, 0x2f022773a766795dULL }, + { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x695329dad2094e8aULL, 0x2f022773a766795dULL }, { 0, 0, 0, 0 } }; @@ -188,6 +188,13 @@ static void test_hash(void) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } + digest = tommy_hash_u32(TEST_STRHASH32[i].seed, buffer_aligned, strlen(TEST_STRHASH32[i].data)); + if (digest != TEST_STRHASH32[i].digest) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed strhash32 test\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } for (i = 0; TEST_HASH64[i].data; ++i) { @@ -456,7 +463,7 @@ static void test_tommy(void) tommy_arrayblkof_init(&arrayblkof, sizeof(unsigned)); for (i = 0; i < TOMMY_SIZE; ++i) { - tommy_array_insert(&array, &node[i]); + tommy_array_insert_tail(&array, &node[i]); tommy_arrayblkof_grow(&arrayblkof, i + 1); *(unsigned*)tommy_arrayblkof_ref(&arrayblkof, i) = i; } @@ -584,7 +591,11 @@ static void test_tommy(void) tommy_tree_insert(&tree, &node[i], (void*)(uintptr_t)(i + 1)); /* try to insert a duplicate, count should not change */ - tommy_tree_insert(&tree, &node[TOMMY_SIZE], (void*)(uintptr_t)1); + if (tommy_tree_insert_unique(&tree, &node[TOMMY_SIZE], (void*)(uintptr_t)1) != (void*)(uintptr_t)1) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } if (tommy_tree_count(&tree) != TOMMY_SIZE) { /* LCOV_EXCL_START */ diff --git a/cmdline/util.c b/cmdline/util.c index 8cfbde5b..2cc659b6 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -7,6 +7,7 @@ #include "util.h" #include "raid/cpu.h" #include "raid/memory.h" +#include "tommyds/tommyhash.h" /****************************************************************************/ /* memory */ @@ -610,112 +611,6 @@ static inline void util_write64(void* ptr, uint64_t v) memcpy(ptr, &v, sizeof(v)); } -/** - * 64*64 -> 128bit multiply function. - * - * @param A Address of 64-bit number. - * @param B Address of 64-bit number. - * - * Calculates 128-bit C = *A * *B. - * - * Overwrites A contents with C's low 64 bits. - * Overwrites B contents with C's high 64 bits. - */ -#if defined(__x86_64__) || defined(_M_X64) -/** - * This is intentionally placed before the __SIZEOF_INT128__ implementation. - * - * This implementation happens to generate more efficient code. - * While this likely depends only on the compiler, as long as it remains faster, - * we will keep it in this order. - */ -static inline void util_mum(uint64_t* A, uint64_t* B) -{ - uint64_t a = *A; - uint64_t b = *B; - uint64_t low, high; - - __asm__ ( - "mulq %3" - : "=a" (low), "=d" (high) - : "a" (a), "rm" (b) - : "cc" - ); - - *A = low; - *B = high; -} -#elif defined(__SIZEOF_INT128__) -static inline void util_mum(uint64_t* A, uint64_t* B) -{ - __uint128_t product = ((__uint128_t)*A) * ((__uint128_t)*B); - *A = (uint64_t)product; - *B = (uint64_t)(product >> 64); -} -#elif defined(__aarch64__) || defined(_M_ARM64) -/** - * This is intentionally placed after the __SIZEOF_INT128__ implementation. - * - * Although the code looks correct, there is a report of this - * implementation failing the self-test on macOS (Darwin 25.4.0, arm64) - * with Apple clang 21.0.0 (clang-2100.0.123.102). - * - * See: https://github.com/amadvance/snapraid/issues/43 - * - * Note that the implementation is the same of OpenSSL: - * - * #elif defined(__aarch64__) && defined(SIXTY_FOUR_BIT_LONG) - * #if defined(__GNUC__) && __GNUC__ >= 2 - * #define BN_UMULT_HIGH(a, b) ({ \ - * register BN_ULONG ret; \ - * asm ("umulh %0,%1,%2" \ - * : "=r"(ret) \ - * : "r"(a), "r"(b)); \ - * ret; }) - * #endif - * #endif - * - * See: https://github.com/openssl/openssl/blob/0463cbf185680e66a3a8960f10cd33001377d28e/crypto/bn/bn_local.h#L439 - */ -static inline void util_mum(uint64_t* A, uint64_t* B) -{ - uint64_t a = *A; - uint64_t b = *B; - uint64_t low = a * b; - uint64_t high; - __asm__ ( - "umulh %0, %1, %2" - : "=r" (high) - : "r" (a), "r" (b) - ); - *A = low; - *B = high; -} -#else -static inline void util_mum(uint64_t* A, uint64_t* B) -{ - uint64_t a = *A; - uint64_t b = *B; - - uint64_t a_lo = (uint32_t)a; - uint64_t a_hi = a >> 32; - uint64_t b_lo = (uint32_t)b; - uint64_t b_hi = b >> 32; - - uint64_t p0 = a_lo * b_lo; - uint64_t p1 = a_lo * b_hi; - uint64_t p2 = a_hi * b_lo; - uint64_t p3 = a_hi * b_hi; - - uint64_t middle = p1 + (p0 >> 32) + (uint32_t)p2; - uint64_t low = (middle << 32) | (uint32_t)p0; - uint64_t high = p3 + (middle >> 32) + (p2 >> 32); - - *A = low; - *B = high; -} -#endif - int util_selftest(void) { /* rotate left 32-bit */ @@ -748,33 +643,33 @@ int util_selftest(void) if (util_rotr64(0x123456789ABCDEF0ULL, 0) != 0x123456789ABCDEF0ULL) return -1; if (util_rotr64(0x123456789ABCDEF0ULL, 64) != 0x123456789ABCDEF0ULL) return -1; - /* util_mum tests */ + /* tommy_mum tests */ uint64_t a, b; /* small values */ a = 3; b = 4; - util_mum(&a, &b); + tommy_mum(&a, &b); if (a != 12 || b != 0) return -1; /* 1 * anything */ a = 0xFFFFFFFFFFFFFFFFULL; b = 1; - util_mum(&a, &b); + tommy_mum(&a, &b); if (a != 0xFFFFFFFFFFFFFFFFULL || b != 0) return -1; /* (2^32) * (2^32) = 2^64 */ a = 1ULL << 32; b = 1ULL << 32; - util_mum(&a, &b); + tommy_mum(&a, &b); if (a != 0 || b != 1) return -1; /* max uint64 * max uint64 */ a = 0xFFFFFFFFFFFFFFFFULL; b = 0xFFFFFFFFFFFFFFFFULL; - util_mum(&a, &b); + tommy_mum(&a, &b); if (a != 1 || b != 0xFFFFFFFFFFFFFFFEULL) return -1; /* larger random-ish test */ a = 0x0123456789ABCDEFULL; b = 0xFEDCBA9876543210ULL; - util_mum(&a, &b); + tommy_mum(&a, &b); if (a != 0x2236D88FE5618CF0ULL || b != 0x121FA00AD77D742ULL) return -1; return 0; diff --git a/tommyds/tommy.c b/tommyds/tommy.c index f3c16d8c..51c2859b 100644 --- a/tommyds/tommy.c +++ b/tommyds/tommy.c @@ -4,10 +4,11 @@ /* redefine the malloc for tommy use */ #define tommy_malloc malloc_nofail #define tommy_calloc calloc_nofail +#define tommy_realloc realloc_nofail #define tommy_free free #include "os/portable.h" -#include "cmdline/support.h" /* for malloc/calloc_nofail() */ +#include "cmdline/support.h" /* for malloc/calloc/realloc_nofail() */ #include "tommyhash.c" #include "tommyarray.c" diff --git a/tommyds/tommyarray.c b/tommyds/tommyarray.c index 90445659..2b9c8261 100644 --- a/tommyds/tommyarray.c +++ b/tommyds/tommyarray.c @@ -3,18 +3,18 @@ #include "tommyarray.h" +#include /* for memset */ + /******************************************************************************/ /* array */ TOMMY_API void tommy_array_init(tommy_array* array) { - tommy_uint_t i; - /* fixed initial size */ array->bucket_bit = TOMMY_ARRAY_BIT; array->bucket_max = (tommy_size_t)1 << array->bucket_bit; array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); - for (i = 1; i < TOMMY_ARRAY_BIT; ++i) + for (tommy_uint_t i = 1; i < TOMMY_ARRAY_BIT; ++i) array->bucket[i] = array->bucket[0]; array->count = 0; @@ -22,26 +22,18 @@ TOMMY_API void tommy_array_init(tommy_array* array) TOMMY_API void tommy_array_done(tommy_array* array) { - tommy_uint_t i; - tommy_free(array->bucket[0]); - for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { + for (tommy_uint_t i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { void** segment = array->bucket[i]; tommy_free(&segment[(tommy_ptrdiff_t)1 << i]); } } -TOMMY_API void tommy_array_grow(tommy_array* array, tommy_size_t count) +TOMMY_API void tommy_array_reserve(tommy_array* array, tommy_size_t size) { - if (array->count >= count) - return; - array->count = count; - - while (count > array->bucket_max) { - void** segment; - + while (size > array->bucket_max) { /* allocate one more segment */ - segment = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); + void** segment = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); /* store it adjusting the offset */ /* cast to ptrdiff_t to ensure to get a negative value */ @@ -52,7 +44,78 @@ TOMMY_API void tommy_array_grow(tommy_array* array, tommy_size_t count) } } -TOMMY_API tommy_size_t tommy_array_memory_usage(tommy_array* array) +TOMMY_API void tommy_array_resize(tommy_array* array, tommy_size_t size) +{ + if (size >= array->count) { + tommy_array_grow(array, size); + return; + } + + /* clear the unused elements to maintain the invariant that unused slots are zero */ + tommy_size_t pos = size; + while (pos < array->count) { + tommy_uint_t bsr = tommy_ilog2(pos | 1); + tommy_size_t seg_end = (bsr < TOMMY_ARRAY_BIT) ? ((tommy_size_t)1 << TOMMY_ARRAY_BIT) : ((tommy_size_t)1 << (bsr + 1)); + tommy_size_t chunk_end = array->count < seg_end ? array->count : seg_end; + + memset(&array->bucket[bsr][pos], 0, (chunk_end - pos) * sizeof(void*)); + pos = chunk_end; + } + + array->count = size; +} + +TOMMY_API void tommy_array_shrink(tommy_array* array) +{ + tommy_uint_t target_bucket_bit = TOMMY_ARRAY_BIT; + + if (array->count > (tommy_size_t)1 << TOMMY_ARRAY_BIT) + target_bucket_bit = tommy_ilog2(array->count - 1) + 1; + + while (array->bucket_bit > target_bucket_bit) { + --array->bucket_bit; + void** segment = array->bucket[array->bucket_bit]; + tommy_free(&segment[(tommy_ptrdiff_t)1 << array->bucket_bit]); + } + + array->bucket_max = (tommy_size_t)1 << array->bucket_bit; +} + +TOMMY_API void tommy_array_foreach(tommy_array* array, tommy_foreach_func* func) +{ + tommy_size_t pos = 0; + + while (pos < array->count) { + tommy_uint_t bsr = tommy_ilog2(pos | 1); + tommy_size_t seg_end = (bsr < TOMMY_ARRAY_BIT) ? ((tommy_size_t)1 << TOMMY_ARRAY_BIT) : ((tommy_size_t)1 << (bsr + 1)); + tommy_size_t chunk_end = array->count < seg_end ? array->count : seg_end; + void** ptr = array->bucket[bsr]; + + while (pos < chunk_end) { + func(ptr[pos]); + ++pos; + } + } +} + +TOMMY_API void tommy_array_foreach_arg(tommy_array* array, tommy_foreach_arg_func* func, void* arg) +{ + tommy_size_t pos = 0; + + while (pos < array->count) { + tommy_uint_t bsr = tommy_ilog2(pos | 1); + tommy_size_t seg_end = (bsr < TOMMY_ARRAY_BIT) ? ((tommy_size_t)1 << TOMMY_ARRAY_BIT) : ((tommy_size_t)1 << (bsr + 1)); + tommy_size_t chunk_end = array->count < seg_end ? array->count : seg_end; + void** ptr = array->bucket[bsr]; + + while (pos < chunk_end) { + func(arg, ptr[pos]); + ++pos; + } + } +} + +TOMMY_API tommy_size_t tommy_array_memory_usage(const tommy_array* array) { return array->bucket_max * (tommy_size_t)sizeof(void*); } diff --git a/tommyds/tommyarray.h b/tommyds/tommyarray.h index f5f4d726..ff92c762 100644 --- a/tommyds/tommyarray.h +++ b/tommyds/tommyarray.h @@ -7,14 +7,14 @@ * This array is able to grow dynamically upon request, without any reallocation. * * The grow operation involves an allocation of a new array segment, without reallocating - * the already used memory, and thus **not increasing** the heap fragmentation. - * This also implies that the address of the stored elements never change. + * the already used memory, and thus not increasing the heap fragmentation. + * This also implies that the address of the stored elements never changes. * * Allocated segments grow in size exponentially. */ -#ifndef __TOMMYARRAY_H -#define __TOMMYARRAY_H +#ifndef TOMMY_ARRAY_H +#define TOMMY_ARRAY_H #include "tommytypes.h" @@ -42,33 +42,83 @@ typedef struct tommy_array_struct { /** * Initializes the array. + * \param array The array. */ TOMMY_API void tommy_array_init(tommy_array* array); /** * Deinitializes the array. + * \param array The array. */ TOMMY_API void tommy_array_done(tommy_array* array); /** - * Grows the size up to the specified value. - * All the new elements in the array are initialized with the 0 value. + * Allocates space for at least the specified number of elements. + * The initialized size and existing elements are unchanged. + * Existing element references remain valid. + * \param array The array. + * \param size Number of elements to reserve space for. */ -TOMMY_API void tommy_array_grow(tommy_array* array, tommy_size_t size); +TOMMY_API void tommy_array_reserve(tommy_array* array, tommy_size_t size); /** - * Gets a reference of the element at the specified position. + * Grows the size up to the specified value. + * All the new elements in the array are initialized with the 0 value. + * \param array The array. + * \param size New size of the array. + */ +tommy_inline void tommy_array_grow(tommy_array* array, tommy_size_t size) +{ + if (size > array->count) { + array->count = size; + + if (size > array->bucket_max) + tommy_array_reserve(array, size); + } +} + +/** + * Changes the initialized size, preserving the common prefix. + * New elements are initialized to zero. + * Reducing the size does not release allocated capacity. + * \param array The array. + * \param size New size of the array. + */ +TOMMY_API void tommy_array_resize(tommy_array* array, tommy_size_t size); + +/** + * Removes all elements, preserving the allocated capacity. + * Call tommy_array_shrink() after clearing to restore the initial capacity. + * Pointed-to elements are not freed. + * The array remains initialized and can be reused immediately. + * \param array The array. + */ +tommy_inline void tommy_array_clear(tommy_array* array) +{ + tommy_array_resize(array, 0); +} + +/** + * Releases unused allocated memory. + * Preserves the size, values, and addresses of existing elements. + * \param array The array. + */ +TOMMY_API void tommy_array_shrink(tommy_array* array); + +/** + * Gets a reference to the element at the specified position. * You must be sure that space for this position is already - * allocated calling tommy_array_grow(). + * allocated by calling tommy_array_grow(). + * \param array The array. + * \param pos Position of the element. + * \return Pointer to the slot at the specified position. */ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) { - tommy_uint_t bsr; - assert(pos < array->count); /* get the highest bit set, in case of all 0, return 0 */ - bsr = tommy_ilog2(pos | 1); + tommy_uint_t bsr = tommy_ilog2(pos | 1); return &array->bucket[bsr][pos]; } @@ -76,7 +126,10 @@ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) /** * Sets the element at the specified position. * You must be sure that space for this position is already - * allocated calling tommy_array_grow(). + * allocated by calling tommy_array_grow(). + * \param array The array. + * \param pos Position of the element. + * \param element Element to set. */ tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element) { @@ -86,7 +139,10 @@ tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* el /** * Gets the element at the specified position. * You must be sure that space for this position is already - * allocated calling tommy_array_grow(). + * allocated by calling tommy_array_grow(). + * \param array The array. + * \param pos Position of the element. + * \return The element at the specified position. */ tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos) { @@ -94,28 +150,129 @@ tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos) } /** - * Grows and inserts a new element at the end of the array. + * Gets the last element. + * The array must not be empty. + * \param array The array. + * \return The last element. */ -tommy_inline void tommy_array_insert(tommy_array* array, void* element) +tommy_inline void* tommy_array_tail(tommy_array* array) +{ + assert(array->count != 0); + return tommy_array_get(array, array->count - 1); +} + +/** + * Grows and inserts a new element at the end of the array. + * \param array The array. + * \param element Element to insert. + */ +tommy_inline void tommy_array_insert_tail(tommy_array* array, void* element) { tommy_size_t pos = array->count; - tommy_array_grow(array, pos + 1); + ++array->count; + if (array->count > array->bucket_max) + tommy_array_reserve(array, array->count); tommy_array_set(array, pos, element); } /** - * Gets the initialized size of the array. + * Removes and returns the last element. + * The array must not be empty. + * The removed slot is cleared, and allocated capacity is preserved. + * \param array The array. + * \return The removed element. */ -tommy_inline tommy_size_t tommy_array_size(tommy_array* array) +tommy_inline void* tommy_array_remove_tail(tommy_array* array) +{ + assert(array->count != 0); + + tommy_size_t pos = array->count - 1; + void** ptr = tommy_array_ref(array, pos); + void* element = *ptr; + + /* keep unused slots zero so growing within capacity needs no initialization */ + *ptr = 0; + array->count = pos; + + return element; +} + +/** + * Checks whether the array is empty. + * \param array The array. + * \return Nonzero if the array is empty, 0 otherwise. + */ +tommy_inline tommy_bool_t tommy_array_empty(const tommy_array* array) +{ + return array->count == 0; +} + +/** + * Gets the initialized size of the array. + * \param array The array. + * \return The initialized size of the array. + */ +tommy_inline tommy_size_t tommy_array_size(const tommy_array* array) { return array->count; } /** - * Gets the size of allocated memory. + * Gets the number of elements that fit without further allocation. + * \param array The array. + * \return The capacity of the array. */ -TOMMY_API tommy_size_t tommy_array_memory_usage(tommy_array* array); +tommy_inline tommy_size_t tommy_array_capacity(const tommy_array* array) +{ + return array->bucket_max; +} + +/** + * Exchanges two initialized arrays without copying their elements. + * Element references remain valid and belong to the other array. + * Passing the same array twice has no effect. + * \param first The first array. + * \param second The second array. + * \note This operation is O(1). + */ +tommy_inline void tommy_array_swap(tommy_array* first, tommy_array* second) +{ + tommy_array tmp = *first; + *first = *second; + *second = tmp; +} + +/** + * Calls the specified function for each element in the array. + * + * You cannot add or remove elements, nor change the size of the array, + * from inside the callback. + * + * If the callback frees the elements, the array retains dangling + * pointers. Call tommy_array_clear() before reusing it, or + * tommy_array_done() to release its storage. + * \param array The array. + * \param func Function to call with each element. + */ +TOMMY_API void tommy_array_foreach(tommy_array* array, tommy_foreach_func* func); + +/** + * Calls the specified function with an argument for each element in the array. + * + * \param array The array. + * \param func Function to call with each element. + * \param arg Argument to pass to the function. + */ +TOMMY_API void tommy_array_foreach_arg(tommy_array* array, tommy_foreach_arg_func* func, void* arg); + +/** + * Gets the size of allocated memory. + * \param array The array. + * \return Total memory allocated by the array, in bytes. + */ +TOMMY_API tommy_size_t tommy_array_memory_usage(const tommy_array* array); #endif + diff --git a/tommyds/tommyarrayblkof.c b/tommyds/tommyarrayblkof.c index da2d43c6..a5b1f070 100644 --- a/tommyds/tommyarrayblkof.c +++ b/tommyds/tommyarrayblkof.c @@ -3,12 +3,16 @@ #include "tommyarrayblkof.h" +#include /* for memset */ + /******************************************************************************/ /* array */ TOMMY_API void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size) { - tommy_array_init(&array->block); + array->block_capacity = (tommy_size_t)1 << TOMMY_ARRAYBLKOF_BIT; + array->block = tommy_cast(unsigned char**, tommy_malloc(array->block_capacity * sizeof(array->block[0]))); + array->block_count = 0; array->element_size = element_size; array->count = 0; @@ -16,44 +20,127 @@ TOMMY_API void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t eleme TOMMY_API void tommy_arrayblkof_done(tommy_arrayblkof* array) { - tommy_size_t i; + for (tommy_size_t i = 0; i < array->block_count; ++i) + tommy_free(array->block[i]); - for (i = 0; i < tommy_array_size(&array->block); ++i) - tommy_free(tommy_array_get(&array->block, i)); - - tommy_array_done(&array->block); + tommy_free(array->block); } -TOMMY_API void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count) +TOMMY_API void tommy_arrayblkof_reserve(tommy_arrayblkof* array, tommy_size_t size) { - tommy_size_t block_max; - tommy_size_t block_mac; - - if (array->count >= count) + if (size <= array->block_count * TOMMY_ARRAYBLKOF_SIZE) return; - array->count = count; - block_max = (count + TOMMY_ARRAYBLKOF_SIZE - 1) / TOMMY_ARRAYBLKOF_SIZE; - block_mac = tommy_array_size(&array->block); + tommy_size_t block_max = (size - 1) / TOMMY_ARRAYBLKOF_SIZE + 1; - if (block_mac < block_max) { - /* grow the block array */ - tommy_array_grow(&array->block, block_max); + if (array->block_capacity < block_max) { + /* only the directory moves; addresses inside existing blocks stay valid */ + tommy_size_t capacity = array->block_capacity; + while (capacity < block_max) + capacity *= 2; + array->block = tommy_cast(unsigned char**, tommy_realloc(array->block, capacity * sizeof(array->block[0]))); + array->block_capacity = capacity; + } - /* allocate new blocks */ - while (block_mac < block_max) { - void** ptr = tommy_cast(void**, tommy_calloc(TOMMY_ARRAYBLKOF_SIZE, array->element_size)); + /* allocate new blocks */ + while (array->block_count < block_max) { + unsigned char* ptr = tommy_cast(unsigned char*, tommy_calloc(TOMMY_ARRAYBLKOF_SIZE, array->element_size)); - /* set the new block */ - tommy_array_set(&array->block, block_mac, ptr); - - ++block_mac; - } + array->block[array->block_count] = ptr; + ++array->block_count; } } -TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array) +TOMMY_API void tommy_arrayblkof_resize(tommy_arrayblkof* array, tommy_size_t size) { - return tommy_array_memory_usage(&array->block) + tommy_array_size(&array->block) * TOMMY_ARRAYBLKOF_SIZE * array->element_size; + if (size >= array->count) { + tommy_arrayblkof_grow(array, size); + return; + } + + /* clear the unused elements to maintain the invariant that unused slots are zero */ + tommy_size_t pos = size; + while (pos < array->count) { + tommy_size_t blk_idx = pos / TOMMY_ARRAYBLKOF_SIZE; + tommy_size_t blk_offset = pos % TOMMY_ARRAYBLKOF_SIZE; + tommy_size_t blk_rem = TOMMY_ARRAYBLKOF_SIZE - blk_offset; + tommy_size_t count_rem = array->count - pos; + tommy_size_t chunk = blk_rem < count_rem ? blk_rem : count_rem; + + memset(array->block[blk_idx] + blk_offset * array->element_size, 0, chunk * array->element_size); + pos += chunk; + } + + array->count = size; +} + +TOMMY_API void tommy_arrayblkof_shrink(tommy_arrayblkof* array) +{ + tommy_size_t block_max = (array->count == 0) ? 0 : ((array->count - 1) / TOMMY_ARRAYBLKOF_SIZE + 1); + + while (array->block_count > block_max) { + --array->block_count; + tommy_free(array->block[array->block_count]); + } + + tommy_size_t min_capacity = (tommy_size_t)1 << TOMMY_ARRAYBLKOF_BIT; + tommy_size_t capacity = min_capacity; + while (capacity < block_max) + capacity *= 2; + if (capacity < array->block_capacity) { + array->block = tommy_cast(unsigned char**, tommy_realloc(array->block, capacity * sizeof(array->block[0]))); + array->block_capacity = capacity; + } +} + +TOMMY_API void tommy_arrayblkof_foreach(tommy_arrayblkof* array, tommy_foreach_func* func) +{ + /* cache the element size to avoid reloading it after each callback */ + tommy_size_t element_size = array->element_size; + tommy_size_t pos = 0; + + while (pos < array->count) { + tommy_size_t blk_idx = pos / TOMMY_ARRAYBLKOF_SIZE; + tommy_size_t chunk = array->count - pos; + unsigned char* ptr = array->block[blk_idx]; + + if (chunk > TOMMY_ARRAYBLKOF_SIZE) + chunk = TOMMY_ARRAYBLKOF_SIZE; + + for (tommy_size_t i = 0; i < chunk; ++i) { + func(ptr); + ptr += element_size; + } + + pos += chunk; + } +} + +TOMMY_API void tommy_arrayblkof_foreach_arg(tommy_arrayblkof* array, tommy_foreach_arg_func* func, void* arg) +{ + /* cache the element size to avoid reloading it after each callback */ + tommy_size_t element_size = array->element_size; + tommy_size_t pos = 0; + + while (pos < array->count) { + tommy_size_t blk_idx = pos / TOMMY_ARRAYBLKOF_SIZE; + tommy_size_t chunk = array->count - pos; + unsigned char* ptr = array->block[blk_idx]; + + if (chunk > TOMMY_ARRAYBLKOF_SIZE) + chunk = TOMMY_ARRAYBLKOF_SIZE; + + for (tommy_size_t i = 0; i < chunk; ++i) { + func(arg, ptr); + ptr += element_size; + } + + pos += chunk; + } +} + +TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(const tommy_arrayblkof* array) +{ + return array->block_capacity * sizeof(array->block[0]) + array->block_count * TOMMY_ARRAYBLKOF_SIZE * array->element_size; } diff --git a/tommyds/tommyarrayblkof.h b/tommyds/tommyarrayblkof.h index 8c9edfa0..7178ae42 100644 --- a/tommyds/tommyarrayblkof.h +++ b/tommyds/tommyarrayblkof.h @@ -2,29 +2,43 @@ // Copyright (C) 2013 Andrea Mazzoleni /** \file - * Dynamic array based on blocks of fixed size. + * Dynamic array based on fixed-size blocks. * - * This array is able to grow dynamically upon request, without any reallocation. + * This array is able to grow dynamically without reallocating stored elements. + * The directory of block pointers may be reallocated, but element addresses + * never change. * - * This is very similar to ::tommy_arrayblk, but it allows to store elements of any + * This is very similar to ::tommy_arrayblk, but it allows storing elements of any * size and not just pointers. * - * Note that in this case tommy_arrayblkof_ref() returns a pointer to the element, - * that should be used for getting and setting elements in the array, - * as generic getter and setter are not available. + * The container allocates space for elements but never copies user data into + * or out of it. tommy_arrayblkof_ref(), tommy_arrayblkof_tail(), and + * tommy_arrayblkof_insert_tail() return an element's address so the caller can + * read or write it directly. + * + * \note Only types with fundamental alignment requirements are supported. + * Storage is allocated using calloc(), which does not guarantee the alignment + * required by over-aligned types (e.g. _Alignas(32) or _Alignas(64)). + * When storing typed elements, use sizeof(type) as the element size to + * preserve alignment for every element. */ -#ifndef __TOMMYARRAYBLKOF_H -#define __TOMMYARRAYBLKOF_H +#ifndef TOMMY_ARRAYBLKOF_H +#define TOMMY_ARRAYBLKOF_H #include "tommytypes.h" -#include "tommyarray.h" #include /* for assert */ +#include /* for memset */ /******************************************************************************/ /* array */ +/** + * Initial and minimal capacity of the block directory expressed as a power of 2. + */ +#define TOMMY_ARRAYBLKOF_BIT 6 + /** * Elements for each block. */ @@ -35,55 +49,214 @@ * \note Don't use internal fields directly, but access the container only using functions. */ typedef struct tommy_arrayblkof_struct { - tommy_array block; /**< Array of blocks. */ + unsigned char** block; /**< Directory of blocks. */ + tommy_size_t block_count; /**< Number of allocated blocks. */ + tommy_size_t block_capacity; /**< Number of directory entries allocated. */ tommy_size_t element_size; /**< Size of the stored element in bytes. */ tommy_size_t count; /**< Number of initialized elements in the array. */ } tommy_arrayblkof; /** * Initializes the array. - * \param element_size Size in byte of the element to store in the array. + * \param array The array. + * \param element_size Size in bytes of the element to store in the array. */ TOMMY_API void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size); /** * Deinitializes the array. + * \param array The array. */ TOMMY_API void tommy_arrayblkof_done(tommy_arrayblkof* array); /** - * Grows the size up to the specified value. - * All the new elements in the array are initialized with the 0 value. + * Allocates space for at least the specified number of elements. + * The initialized size and existing elements are unchanged. + * Existing element references remain valid. + * \param array The array. + * \param size Number of elements to reserve space for. */ -TOMMY_API void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size); +TOMMY_API void tommy_arrayblkof_reserve(tommy_arrayblkof* array, tommy_size_t size); /** - * Gets a reference of the element at the specified position. + * Grows the size up to the specified value. + * All the new elements in the array are initialized with the 0 value. + * \param array The array. + * \param size New size of the array. + */ +tommy_inline void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size) +{ + if (size > array->count) { + array->count = size; + + if (size > array->block_count * TOMMY_ARRAYBLKOF_SIZE) + tommy_arrayblkof_reserve(array, size); + } +} + +/** + * Changes the initialized size, preserving the common prefix. + * New elements are initialized to zero. + * Reducing the size does not release allocated capacity. + * \param array The array. + * \param size New size of the array. + */ +TOMMY_API void tommy_arrayblkof_resize(tommy_arrayblkof* array, tommy_size_t size); + +/** + * Removes all elements, preserving the allocated capacity. + * Call tommy_arrayblkof_shrink() after clearing to restore the initial capacity. + * The array remains initialized and can be reused immediately. + * \param array The array. + */ +tommy_inline void tommy_arrayblkof_clear(tommy_arrayblkof* array) +{ + tommy_arrayblkof_resize(array, 0); +} + +/** + * Releases unused allocated memory. + * Preserves the size, values, and addresses of existing elements. + * \param array The array. + */ +TOMMY_API void tommy_arrayblkof_shrink(tommy_arrayblkof* array); + +/** + * Gets a reference to the element at the specified position. * You must be sure that space for this position is already - * allocated calling tommy_arrayblkof_grow(). + * allocated by calling tommy_arrayblkof_grow(). + * \param array The array. + * \param pos Position of the element. + * \return Pointer to the slot at the specified position. */ tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos) { - unsigned char* base; - assert(pos < array->count); - base = tommy_cast(unsigned char*, tommy_array_get(&array->block, pos / TOMMY_ARRAYBLKOF_SIZE)); + return array->block[pos / TOMMY_ARRAYBLKOF_SIZE] + (pos % TOMMY_ARRAYBLKOF_SIZE) * array->element_size; +} - return base + (pos % TOMMY_ARRAYBLKOF_SIZE) * array->element_size; +/** + * Gets a reference to the last element. + * The array must not be empty. + * \param array The array. + * \return Pointer to the last element. + */ +tommy_inline void* tommy_arrayblkof_tail(tommy_arrayblkof* array) +{ + assert(array->count != 0); + return tommy_arrayblkof_ref(array, array->count - 1); +} + +/** + * Adds a zero-initialized element and returns its reference. + * \param array The array. + * \return Pointer to the new element. + */ +tommy_inline void* tommy_arrayblkof_insert_tail(tommy_arrayblkof* array) +{ + tommy_size_t pos = array->count; + + ++array->count; + if (array->count > array->block_count * TOMMY_ARRAYBLKOF_SIZE) + tommy_arrayblkof_reserve(array, array->count); + + return tommy_arrayblkof_ref(array, pos); +} + +/** + * Removes the last element without copying it. + * The array must not be empty. The removed slot is cleared. + * \param array The array. + */ +tommy_inline void tommy_arrayblkof_remove_tail(tommy_arrayblkof* array) +{ + assert(array->count != 0); + + tommy_size_t pos = array->count - 1; + void* ptr = tommy_arrayblkof_ref(array, pos); + + /* keep unused slots zero so growing within capacity needs no initialization */ + memset(ptr, 0, array->element_size); + array->count = pos; +} + +/** + * Checks whether the array is empty. + * \param array The array. + * \return Nonzero if the array is empty, 0 otherwise. + */ +tommy_inline tommy_bool_t tommy_arrayblkof_empty(const tommy_arrayblkof* array) +{ + return array->count == 0; } /** * Gets the initialized size of the array. + * \param array The array. + * \return The initialized size of the array. */ -tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array) +tommy_inline tommy_size_t tommy_arrayblkof_size(const tommy_arrayblkof* array) { return array->count; } /** - * Gets the size of allocated memory. + * Gets the number of elements that fit without further allocation. + * \param array The array. + * \return The capacity of the array. */ -TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array); +tommy_inline tommy_size_t tommy_arrayblkof_capacity(const tommy_arrayblkof* array) +{ + return array->block_count * TOMMY_ARRAYBLKOF_SIZE; +} + +/** + * Exchanges two initialized arrays without copying their elements. + * Element references remain valid and belong to the other array. + * Passing the same array twice has no effect. + * \param first The first array. + * \param second The second array. + * \note This operation is O(1). + */ +tommy_inline void tommy_arrayblkof_swap(tommy_arrayblkof* first, tommy_arrayblkof* second) +{ + /* keep record size with the storage it describes */ + tommy_arrayblkof tmp = *first; + *first = *second; + *second = tmp; +} + +/** + * Calls the specified function for each element in the array. + * + * You cannot add or remove elements, nor change the size of the array, + * from inside the callback. + * + * Elements are stored inside the array: passing free() as the callback + * is invalid. A callback may release resources owned by each element, + * but must not free the element itself. Call tommy_arrayblkof_clear() before + * reusing it, or tommy_arrayblkof_done() to release its storage. + * \param array The array. + * \param func Function to call with each element. + */ +TOMMY_API void tommy_arrayblkof_foreach(tommy_arrayblkof* array, tommy_foreach_func* func); + +/** + * Calls the specified function with an argument for each element in the array. + * + * \param array The array. + * \param func Function to call with each element. + * \param arg Argument to pass to the function. + */ +TOMMY_API void tommy_arrayblkof_foreach_arg(tommy_arrayblkof* array, tommy_foreach_arg_func* func, void* arg); + +/** + * Gets the size of allocated memory. + * \param array The array. + * \return Total memory allocated by the array, in bytes. + */ +TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(const tommy_arrayblkof* array); #endif + diff --git a/tommyds/tommychain.h b/tommyds/tommychain.h index c864dccd..d1c293f6 100644 --- a/tommyds/tommychain.h +++ b/tommyds/tommychain.h @@ -9,8 +9,8 @@ * Do not use this directly. Use lists instead. */ -#ifndef __TOMMYCHAIN_H -#define __TOMMYCHAIN_H +#ifndef TOMMY_CHAIN_H +#define TOMMY_CHAIN_H #include "tommytypes.h" @@ -65,31 +65,39 @@ tommy_inline void tommy_chain_concat(tommy_node* first_tail, tommy_node* second_ /** * Merges two chains. * \param first First chain (will contain the result). - * \param second Second chain (will be empty after the merge). + * \param second Second chain (consumed by the merge; its descriptor is not cleared + * and must not be used as an independent chain afterward). * \param cmp Comparison function. */ -tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) +tommy_inline void tommy_chain_merge(tommy_chain* tommy_restrict first, tommy_chain* tommy_restrict second, tommy_compare_func* cmp) { - tommy_node* first_i = first->head; + /* + * comparators must not change the chains, so cache these endpoints across calls. + * also caching first_tail caused register spills with GCC and Clang on x86-64. + */ + tommy_node* first_head = first->head; + tommy_node* second_tail = second->tail; + tommy_node* first_i = first_head; tommy_node* second_i = second->head; /* merge */ while (1) { if (cmp(first_i->data, second_i->data) > 0) { tommy_node* next = second_i->next; - if (first_i == first->head) { + if (first_i == first_head) { tommy_chain_concat(second_i, first_i); - first->head = second_i; + first_head = second_i; + first->head = first_head; } else { tommy_chain_splice(first_i->prev, first_i, second_i, second_i); } - if (second_i == second->tail) + if (second_i == second_tail) break; second_i = next; } else { if (first_i == first->tail) { tommy_chain_concat(first_i, second_i); - first->tail = second->tail; + first->tail = second_tail; break; } first_i = first_i->next; @@ -98,13 +106,36 @@ tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tom } /** - * Merges two chains managing special degenerated cases. - * It's functionally equivalent to tommy_chain_merge() but faster with already ordered chains. - * \param first First chain (will contain the result). - * \param second Second chain (will be empty after the merge). + * Merges two chains containing one node each. + * \param first First singleton chain (will contain the result). + * \param second Second singleton chain (consumed by the merge; its descriptor is not cleared + * and must not be used as an independent chain afterward). * \param cmp Comparison function. */ -tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) +tommy_inline void tommy_chain_merge_singleton(tommy_chain* tommy_restrict first, tommy_chain* tommy_restrict second, tommy_compare_func* cmp) +{ + tommy_node* first_node = first->head; + tommy_node* second_node = second->head; + + /* keep equal elements in their original order for a stable sort */ + if (cmp(first_node->data, second_node->data) <= 0) { + tommy_chain_concat(first_node, second_node); + first->tail = second_node; + } else { + tommy_chain_concat(second_node, first_node); + first->head = second_node; + } +} + +/** + * Merges two chains managing special degenerate cases. + * It's functionally equivalent to tommy_chain_merge() but faster with already ordered chains. + * \param first First chain (will contain the result). + * \param second Second chain (consumed by the merge; its descriptor is not cleared + * and must not be used as an independent chain afterward). + * \param cmp Comparison function. + */ +tommy_inline void tommy_chain_merge_degenerated(tommy_chain* tommy_restrict first, tommy_chain* tommy_restrict second, tommy_compare_func* cmp) { /* identify the condition first <= second */ if (cmp(first->tail->data, second->head->data) <= 0) { @@ -126,9 +157,9 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* /** * Sorts a chain. - * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, + * It's a stable merge sort using power of 2 buckets, with O(n log n) complexity, * similar to the one used in the SGI STL libraries and in the Linux Kernel, - * but faster on degenerated cases like already ordered lists. + * but faster on degenerate cases like already ordered lists. * * SGI STL stl_list.h * http://www.sgi.com/tech/stl/stl_list.h @@ -136,43 +167,48 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* * Linux Kernel lib/list_sort.c * http://lxr.linux.no/#linux+v2.6.36/lib/list_sort.c * - * \param chain Chain to sort. + * \param chain The chain. * \param cmp Comparison function. */ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* cmp) { /* * Bit buckets of chains. - * Each bucket contains 2^i nodes or it's empty. + * Each bucket contains 2^(i+1) nodes or it's empty. + * The bucket containing an unpaired final node has one node less. * The chain at address TOMMY_BIT_MAX is an independent variable operating as "carry". * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. */ tommy_chain bit[TOMMY_SIZE_BIT + 1]; - /** + /* * Value stored inside the bit bucket. * It's used to know which bucket is empty or full. */ - tommy_size_t counter; + tommy_size_t counter = 0; tommy_node* node = chain->head; tommy_node* tail = chain->tail; - tommy_size_t mask; - tommy_size_t i; - counter = 0; while (1) { - tommy_node* next; - tommy_chain* last; - /* carry bit to add */ - last = &bit[TOMMY_SIZE_BIT]; + tommy_chain* last = &bit[TOMMY_SIZE_BIT]; bit[TOMMY_SIZE_BIT].head = node; bit[TOMMY_SIZE_BIT].tail = node; - next = node->next; + tommy_node* next = node->next; + + /* start with a sorted pair to halve the number of bucket insertions */ + if (node != tail) { + node = next; + /* save the next input node before merging can change the links */ + next = node->next; + tommy_chain second = { node, node }; + tommy_chain_merge_singleton(last, &second, cmp); + } /* add the bit, propagating the carry */ - i = 0; - mask = counter; + tommy_size_t i = 0; + tommy_size_t mask = counter; + while ((mask & 1) != 0) { tommy_chain_merge_degenerated(&bit[i], last, cmp); mask >>= 1; @@ -192,8 +228,8 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* } /* merge the buckets */ - i = tommy_ctz(counter); - mask = counter >> i; + tommy_size_t i = tommy_ctz(counter); + tommy_size_t mask = counter >> i; while (mask != 1) { mask >>= 1; if (mask & 1) @@ -207,3 +243,4 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* } #endif + diff --git a/tommyds/tommyhash.c b/tommyds/tommyhash.c index f6b022b3..fc4fd276 100644 --- a/tommyds/tommyhash.c +++ b/tommyds/tommyhash.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: BSD-2-Clause +// SPDX-License-Identifier: BSD-2-Clause AND MIT // Copyright (C) 2010 Andrea Mazzoleni #include "tommyhash.h" @@ -13,222 +13,178 @@ * They are needed only if BigEndian. */ #if defined(__GNUC__) -#define tommy_swap32(x) __builtin_bswap32(x) +#define tommy_imp_hash_swap32(x) __builtin_bswap32(x) #else -tommy_inline tommy_uint32_t tommy_swap32(tommy_uint32_t v) +tommy_inline tommy_uint32_t tommy_imp_hash_swap32(tommy_uint32_t v) { return ((v & 0xFF000000) >> 24) | - ((v & 0x00FF0000) >> 8) | - ((v & 0x0000FF00) << 8) | - ((v & 0x000000FF) << 24); + ((v & 0x00FF0000) >> 8) | + ((v & 0x0000FF00) << 8) | + ((v & 0x000000FF) << 24); } #endif -tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr) +tommy_inline tommy_uint32_t tommy_imp_hash_le_uint32_read(const void* ptr) { tommy_uint32_t v; memcpy(&v, ptr, sizeof(v)); #if defined(WORDS_BIGENDIAN) || defined(__BIG_ENDIAN__) || \ (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - v = tommy_swap32(v); + v = tommy_imp_hash_swap32(v); #endif return v; } -#define tommy_rot(x, k) \ - (((x) << (k)) | ((x) >> (32 - (k)))) +/******************************************************************************/ +/* a5hash */ -#define tommy_mix(a, b, c) \ - do { \ - a -= c; a ^= tommy_rot(c, 4); c += b; \ - b -= a; b ^= tommy_rot(a, 6); a += c; \ - c -= b; c ^= tommy_rot(b, 8); b += a; \ - a -= c; a ^= tommy_rot(c, 16); c += b; \ - b -= a; b ^= tommy_rot(a, 19); a += c; \ - c -= b; c ^= tommy_rot(b, 4); b += a; \ - } while (0) +/** + * a5hash32 and a5hash (64-bit) by Aleksey Vaneev, from https://github.com/avaneev/a5hash. + * Adapted from version 5.25 with little-endian word reads. + * The 32-bit variant additionally mixes the expanded seed asymmetrically into + * h[2] and h[3] before processing the input and produces different hash values. + * Copyright (c) 2025 Aleksey Vaneev. Released under the MIT license, see LICENSES/MIT. + */ -#define tommy_final(a, b, c) \ - do { \ - c ^= b; c -= tommy_rot(b, 14); \ - a ^= c; a -= tommy_rot(c, 11); \ - b ^= a; b -= tommy_rot(a, 25); \ - c ^= b; c -= tommy_rot(b, 16); \ - a ^= c; a -= tommy_rot(c, 4); \ - b ^= a; b -= tommy_rot(a, 14); \ - c ^= b; c -= tommy_rot(b, 24); \ - } while (0) +tommy_inline void tommy_mul(tommy_uint32_t* lo, tommy_uint32_t* hi) +{ + /* both operands are 32-bit; the full product supplies two state words */ + tommy_uint64_t v = (tommy_uint64_t)*lo * *hi; + *lo = (tommy_uint32_t)v; + *hi = v >> 32; +} TOMMY_API tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len) { const unsigned char* key = tommy_cast(const unsigned char*, void_key); - tommy_uint32_t a, b, c; + tommy_uint32_t val01 = 0x55555555; + tommy_uint32_t val10 = 0xaaaaaaaa; + tommy_uint32_t h[4] = { 0x243f6a88, 0x85a308d3, 0xfb0bd3ea, 0x0f58fd47 }; + tommy_uint32_t a, b; + tommy_size_t len = key_len; - a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val; + h[0] ^= key_len; + h[1] ^= key_len; +#if TOMMY_SIZE_BIT == 64 + h[2] = (key_len >> 32) ^ 0x452821e6; + h[3] = (key_len >> 32) ^ 0x38d01377; + tommy_mul(&h[2], &h[3]); +#endif + h[0] ^= init_val & val01; + h[1] ^= init_val & val10; + tommy_mul(&h[0], &h[1]); - while (key_len > 12) { - a += tommy_le_uint32_read(key + 0); - b += tommy_le_uint32_read(key + 4); - c += tommy_le_uint32_read(key + 8); + /* + * Mix the expanded seed asymmetrically into h[2] and h[3] so that short keys + * skipping the main loop still depend on the seed, and neither h[0] nor h[1] + * cancels out when folding h[0] ^= h[2] and h[1] ^= h[3] at the end. + */ + h[2] ^= h[0] ^ h[1]; + h[3] ^= h[0]; - tommy_mix(a, b, c); + if (len <= 16) { + if (len >= 4) { + const unsigned char* tail = key + len - 4; + a = tommy_imp_hash_le_uint32_read(key); + b = tommy_imp_hash_le_uint32_read(tail); + if (len >= 9) { + tommy_size_t offset = (len / 8) * 4; + h[2] += tommy_imp_hash_le_uint32_read(key + offset); + h[3] += tommy_imp_hash_le_uint32_read(tail - offset); + tommy_mul(&h[2], &h[3]); + } + } else { + a = 0; + b = 0; + if (len != 0) { + a = key[0]; + if (len >= 2) + a |= key[1] << 8; + if (len == 3) + a |= key[2] << 16; + } + } + } else { + val01 ^= h[0]; + val10 ^= h[1]; + do { + tommy_uint32_t s1 = h[0]; + tommy_uint32_t s4 = h[3]; + h[0] += tommy_imp_hash_le_uint32_read(key); + h[1] += tommy_imp_hash_le_uint32_read(key + 4); + tommy_mul(&h[0], &h[1]); + h[2] += tommy_imp_hash_le_uint32_read(key + 8); + h[3] += tommy_imp_hash_le_uint32_read(key + 12); + tommy_mul(&h[2], &h[3]); + h[0] += val01; + h[1] += s4; + h[2] += s1; + h[3] += val10; + key += 16; + len -= 16; + } while (len > 16); - key_len -= 12; - key += 12; + /* the last words may overlap the previous block, but stay within the input */ + const unsigned char* tail = key + len; + a = tommy_imp_hash_le_uint32_read(tail - 8); + b = tommy_imp_hash_le_uint32_read(tail - 4); + if (len >= 9) { + h[2] += tommy_imp_hash_le_uint32_read(tail - 16); + h[3] += tommy_imp_hash_le_uint32_read(tail - 12); + tommy_mul(&h[2], &h[3]); + } } - switch (key_len) { - case 0 : - return c; /* used only when called with a zero length */ - case 12 : - c += tommy_le_uint32_read(key + 8); - b += tommy_le_uint32_read(key + 4); - a += tommy_le_uint32_read(key + 0); - break; - case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ - case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ - case 9 : c += key[8]; /* fallthrough */ - case 8 : - b += tommy_le_uint32_read(key + 4); - a += tommy_le_uint32_read(key + 0); - break; - case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ - case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ - case 5 : b += key[4]; /* fallthrough */ - case 4 : - a += tommy_le_uint32_read(key + 0); - break; - case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ - case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ - case 1 : a += key[0]; /* fallthrough */ - } - - tommy_final(a, b, c); - - return c; + h[0] ^= h[2]; + h[1] ^= h[3]; + h[0] += a; + h[1] += b; + tommy_mul(&h[0], &h[1]); + h[0] ^= val01; + tommy_mul(&h[0], &h[1]); + return h[0] ^ h[1]; } TOMMY_API tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len) { const unsigned char* key = tommy_cast(const unsigned char*, void_key); - tommy_uint32_t a, b, c; + tommy_uint64_t val01 = 0x5555555555555555ULL; + tommy_uint64_t val10 = 0xaaaaaaaaaaaaaaaaULL; + tommy_uint64_t h[2] = { (0x452821e638d01377ULL ^ key_len) ^ (init_val & val10), (0x243f6a8885a308d3ULL ^ key_len) ^ (init_val & val01) }; + tommy_size_t len = key_len; - a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff); - c += init_val >> 32; - - while (key_len > 12) { - a += tommy_le_uint32_read(key + 0); - b += tommy_le_uint32_read(key + 4); - c += tommy_le_uint32_read(key + 8); - - tommy_mix(a, b, c); - - key_len -= 12; - key += 12; + tommy_mum(&h[0], &h[1]); + if (len > 16) { + val01 ^= h[0]; + val10 ^= h[1]; + do { + /* each 64-bit operand puts the first little-endian word in its high half */ + h[0] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key) << 32 ^ tommy_imp_hash_le_uint32_read(key + 4); + h[1] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key + 8) << 32 ^ tommy_imp_hash_le_uint32_read(key + 12); + tommy_mum(&h[0], &h[1]); + h[0] += val01; + h[1] += val10; + key += 16; + len -= 16; + } while (len > 16); } - switch (key_len) { - case 0 : - return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */ - case 12 : - c += tommy_le_uint32_read(key + 8); - b += tommy_le_uint32_read(key + 4); - a += tommy_le_uint32_read(key + 0); - break; - case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ - case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ - case 9 : c += key[8]; /* fallthrough */ - case 8 : - b += tommy_le_uint32_read(key + 4); - a += tommy_le_uint32_read(key + 0); - break; - case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ - case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ - case 5 : b += key[4]; /* fallthrough */ - case 4 : - a += tommy_le_uint32_read(key + 0); - break; - case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ - case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ - case 1 : a += key[0]; /* fallthrough */ + if (len >= 4) { + const unsigned char* tail = key + len - 4; + tommy_size_t offset = (len / 8) * 4; + h[0] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key) << 32 | tommy_imp_hash_le_uint32_read(tail); + h[1] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key + offset) << 32 | tommy_imp_hash_le_uint32_read(tail - offset); + } else if (len != 0) { + h[0] ^= key[0]; + if (len >= 2) + h[0] ^= key[1] << 8; + if (len == 3) + h[0] ^= key[2] << 16; } - tommy_final(a, b, c); - - return c + ((tommy_uint64_t)b << 32); -} - -TOMMY_API tommy_uint32_t tommy_strhash_u32(tommy_uint32_t init_val, const void* void_key) -{ - const unsigned char* key = tommy_cast(const unsigned char*, void_key); - tommy_uint32_t a, b, c; - tommy_uint32_t m[3] = { 0xff, 0xff00, 0xff0000 }; - - a = b = c = 0xdeadbeef + init_val; - /* this is different than original lookup3 and the result won't match */ - - while (1) { - tommy_uint32_t v = tommy_le_uint32_read(key); - - if (tommy_haszero_u32(v)) { - if (v & m[0]) { - a += v & m[0]; - if (v & m[1]) { - a += v & m[1]; - if (v & m[2]) - a += v & m[2]; - } - } - - break; - } - - a += v; - - v = tommy_le_uint32_read(key + 4); - - if (tommy_haszero_u32(v)) { - if (v & m[0]) { - b += v & m[0]; - if (v & m[1]) { - b += v & m[1]; - if (v & m[2]) - b += v & m[2]; - } - } - - break; - } - - b += v; - - v = tommy_le_uint32_read(key + 8); - - if (tommy_haszero_u32(v)) { - if (v & m[0]) { - c += v & m[0]; - if (v & m[1]) { - c += v & m[1]; - if (v & m[2]) - c += v & m[2]; - } - } - - break; - } - - c += v; - - tommy_mix(a, b, c); - - key += 12; - } - - /* for lengths that are multipliers of 12 we already have called mix */ - /* this is different than the original lookup3 and the result won't match */ - - tommy_final(a, b, c); - - return c; + tommy_mum(&h[0], &h[1]); + h[0] ^= val01; + tommy_mum(&h[0], &h[1]); + return h[0] ^ h[1]; } diff --git a/tommyds/tommyhash.h b/tommyds/tommyhash.h index 763f5a48..a0dd6043 100644 --- a/tommyds/tommyhash.h +++ b/tommyds/tommyhash.h @@ -2,79 +2,145 @@ // Copyright (C) 2010 Andrea Mazzoleni /** \file - * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. + * Hash functions for use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. + * + * This module provides hash functions designed for in-memory data structures. + * ::tommy_hash_u32 and ::tommy_strhash_u32 use a modified a5hash32, while + * ::tommy_hash_u64 and ::tommy_strhash_u64 use a5hash (64-bit). + * Reversible integer hash functions are also provided for 32-bit and 64-bit keys. + * + * a5hash32 is intended for 32-bit platforms and uses 32-bit operands with + * full 64-bit multiplication results. a5hash uses 64-bit operands with + * full 128-bit multiplication results. Both algorithms pass SMHasher3 tests. + * + * Prefer the 64-bit version whenever possible, as it provides better resistance + * to collisions, particularly with specially crafted inputs. However, neither + * version is designed to resist deliberate collision attacks. If resistance to + * such attacks is required, use a dedicated hash function designed for that purpose. + * + * All hash functions are intended for runtime in-memory use only and not + * for persistent storage, as hash compatibility across different versions of the + * library is not guaranteed. String and sized buffer hash functions produce + * identical results for 0-terminated strings. */ -#ifndef __TOMMYHASH_H -#define __TOMMYHASH_H +#ifndef TOMMY_HASH_H +#define TOMMY_HASH_H #include "tommytypes.h" +#include /* for strlen */ /******************************************************************************/ /* hash */ /** - * Hash function with a 32 bits result. - * Implementation of the Robert Jenkins "lookup3" hash 32 bits version, - * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). + * Hash function with a 32-bit result. + * Modified implementation of a5hash32 by Aleksey Vaneev, + * from https://github.com/avaneev/a5hash * - * This hash is designed to provide a good overall performance on all platforms, - * including 32 bits. If you target only 64 bits, you can use faster hashes, - * like SpookyHash or FarmHash. + * The expanded seed is mixed asymmetrically into the two additional state words + * before processing the input. This preserves seed dependence for short keys + * that skip the main loop and avoids cancellation of either seed state word + * in the final XOR fold. The resulting hash values differ from upstream a5hash32. + * + * This algorithm is compact, fast on short inputs, and passes SMHasher3 tests. * * \param init_val Initialization value. * Using a different initialization value, you can generate a completely different set of hash values. * Use 0 if not relevant. - * \param void_key Pointer to the data to hash. + * \param void_key Pointer to the data to hash. Must not be 0, even if key_len is 0. * \param key_len Size of the data to hash. + * \return The 32-bit hash value. * \note - * This function is endianness independent. - * \return The hash value of 32 bits. + * This function is endianness independent and alignment independent. + * It produces identical hash results to ::tommy_strhash_u32 for 0-terminated strings. + * Compatibility across different versions of the library is not guaranteed: + * this hash is intended for in-memory runtime use only and not for persistent storage. + * It is non-cryptographic and is not designed to protect against malicious collision attacks. */ TOMMY_API tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); /** - * Hash function with a 64 bits result. - * Implementation of the Robert Jenkins "lookup3" hash 64 bits version, - * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2(). + * Hash function with a 64-bit result. + * Implementation of a5hash (64-bit) by Aleksey Vaneev, + * from https://github.com/avaneev/a5hash * - * This hash is designed to provide a good overall performance on all platforms, - * including 32 bits. If you target only 64 bits, you can use faster hashes, - * like SpookyHash or FarmHash. + * This algorithm is compact, fast on short inputs, and passes SMHasher3 tests. * * \param init_val Initialization value. * Using a different initialization value, you can generate a completely different set of hash values. * Use 0 if not relevant. - * \param void_key Pointer to the data to hash. + * \param void_key Pointer to the data to hash. Must not be 0, even if key_len is 0. * \param key_len Size of the data to hash. + * \return The 64-bit hash value. * \note - * This function is endianness independent. - * \return The hash value of 64 bits. + * This function is endianness independent and alignment independent. + * It produces identical hash results to ::tommy_strhash_u64 for 0-terminated strings. + * Compatibility across different versions of the library is not guaranteed: + * this hash is intended for in-memory runtime use only and not for persistent storage. + * It is non-cryptographic and is not designed to protect against malicious collision attacks. */ TOMMY_API tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); /** - * String hash function with a 32 bits result. - * Implementation is based on Robert Jenkins "lookup3" hash 32 bits version, - * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). + * String hash function with a 32-bit result. + * Modified implementation of a5hash32 by Aleksey Vaneev, + * from https://github.com/avaneev/a5hash * - * This hash is designed to handle strings with an unknown length. If you - * know the string length, the other hash functions are surely faster. + * This hash is designed to handle strings with an unknown length. + * It is implemented inline using strlen() followed by ::tommy_hash_u32. + * This gives better performance than scanning for the terminator while hashing. + * It uses the same modified seed mixing and differs from upstream a5hash32. * * \param init_val Initialization value. * Using a different initialization value, you can generate a completely different set of hash values. * Use 0 if not relevant. - * \param void_key Pointer to the string to hash. It has to be 0 terminated. + * \param void_key Pointer to the string to hash. It has to be 0-terminated. + * \return The 32-bit hash value. * \note - * This function is endianness independent. - * \return The hash value of 32 bits. + * This function is endianness independent and alignment independent. + * It produces identical hash results to ::tommy_hash_u32 for 0-terminated strings. + * Compatibility across different versions of the library is not guaranteed: + * this hash is intended for in-memory runtime use only and not for persistent storage. + * It is non-cryptographic and is not designed to protect against malicious collision attacks. */ -TOMMY_API tommy_uint32_t tommy_strhash_u32(tommy_uint32_t init_val, const void* void_key); +tommy_inline tommy_uint32_t tommy_strhash_u32(tommy_uint32_t init_val, const void* void_key) +{ + return tommy_hash_u32(init_val, void_key, strlen(tommy_cast(const char*, void_key))); +} /** - * Integer reversible hash function for 32 bits. + * String hash function with a 64-bit result. + * Implementation of a5hash (64-bit) by Aleksey Vaneev, + * from https://github.com/avaneev/a5hash + * + * This hash is designed to handle strings with an unknown length. + * It is implemented inline using strlen() followed by ::tommy_hash_u64. + * This gives better performance than scanning for the terminator while hashing. + * + * \param init_val Initialization value. + * Using a different initialization value, you can generate a completely different set of hash values. + * Use 0 if not relevant. + * \param void_key Pointer to the string to hash. It has to be 0-terminated. + * \return The 64-bit hash value. + * \note + * This function is endianness independent and alignment independent. + * It produces identical hash results to ::tommy_hash_u64 for 0-terminated strings. + * Compatibility across different versions of the library is not guaranteed: + * this hash is intended for in-memory runtime use only and not for persistent storage. + * It is non-cryptographic and is not designed to protect against malicious collision attacks. + */ +tommy_inline tommy_uint64_t tommy_strhash_u64(tommy_uint64_t init_val, const void* void_key) +{ + return tommy_hash_u64(init_val, void_key, strlen(tommy_cast(const char*, void_key))); +} + +/** + * Reversible 32-bit integer hash function. * Implementation of the Robert Jenkins "4-byte Integer Hashing", * from http://burtleburtle.net/bob/hash/integer.html + * \param key The 32-bit integer to hash. + * \return The 32-bit hash value. */ tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) { @@ -90,9 +156,11 @@ tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) } /** - * Integer reversible hash function for 64 bits. + * Reversible 64-bit integer hash function. * Implementation of the Thomas Wang "Integer Hash Function", * from http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm + * \param key The 64-bit integer to hash. + * \return The 64-bit hash value. */ tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) { @@ -107,4 +175,146 @@ tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) return key; } +/** + * Reversible 32-bit integer hash function using multiplications. + * Implementation of the MurmurHash3 fmix32 finalizer by Austin Appleby, + * from https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp + * \param key The 32-bit integer to hash. + * \return The 32-bit hash value. + * \note Requires an architecture with fast 32-bit integer multiplication + * for good performance. See tommy_inthash_u32() for an alternative. + */ +tommy_inline tommy_uint32_t tommy_mulhash_u32(tommy_uint32_t key) +{ + key ^= key >> 16; + key *= 0x85ebca6bU; + key ^= key >> 13; + key *= 0xc2b2ae35U; + key ^= key >> 16; + + return key; +} + +/** + * Reversible 64-bit integer hash function using multiplications. + * Implementation of the SplitMix64 finalizer, + * from https://prng.di.unimi.it/splitmix64.c + * \param key The 64-bit integer to hash. + * \return The 64-bit hash value. + * \note Requires an architecture with fast 64-bit integer multiplication + * for good performance. It is generally not recommended on 32-bit architectures, + * where these multiplications are expensive. Consider tommy_inthash_u64() instead. + */ +tommy_inline tommy_uint64_t tommy_mulhash_u64(tommy_uint64_t key) +{ + key ^= key >> 30; + key *= 0xbf58476d1ce4e5b9ULL; + key ^= key >> 27; + key *= 0x94d049bb133111ebULL; + key ^= key >> 31; + + return key; +} + +/******************************************************************************/ +/* multiplication */ + +/** + * Multiplies two 64-bit unsigned numbers to obtain the full 128-bit product. + * + * \param A Address of the first operand, overwritten with the low 64 bits of the product. + * \param B Address of the second operand, overwritten with the high 64 bits of the product. + */ +#if defined(_MSC_VER) && defined(_M_X64) +/* + * MSVC x64 does not support inline assembly, so use its full-product intrinsic. + */ +#pragma intrinsic(_umul128) +tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) +{ + tommy_uint64_t high; + tommy_uint64_t low = _umul128(*A, *B, &high); + *A = low; + *B = high; +} +#elif defined(_MSC_VER) && defined(_M_ARM64) +#pragma intrinsic(__umulh) +tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) +{ + tommy_uint64_t a = *A; + tommy_uint64_t b = *B; + *A = a * b; + *B = __umulh(a, b); +} +#elif defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) +/** + * Keep this before __SIZEOF_INT128__ because the explicit mulq generates more efficient code. + */ +tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) +{ + tommy_uint64_t a = *A; + tommy_uint64_t b = *B; + tommy_uint64_t low, high; + + /* require a register to avoid Clang spilling the operand before mulq */ + __asm__ ( + "mulq %3" + : "=a" (low), "=d" (high) + : "a" (a), "r" (b) + : "cc" + ); + + *A = low; + *B = high; +} +#elif defined(__SIZEOF_INT128__) +tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) +{ + __uint128_t product = (__uint128_t)*A * *B; + *A = product; + *B = product >> 64; +} +#elif defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) +/* + * Keep this after __SIZEOF_INT128__ because the assembly path failed optimized + * hash tests on Apple clang 21.0.0. See https://github.com/amadvance/snapraid/issues/43. + */ +tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) +{ + tommy_uint64_t a = *A; + tommy_uint64_t b = *B; + tommy_uint64_t low = a * b; + tommy_uint64_t high; + __asm__ ( + "umulh %0, %1, %2" + : "=r" (high) + : "r" (a), "r" (b) + ); + *A = low; + *B = high; +} +#else +tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) +{ + tommy_uint64_t a = *A; + tommy_uint64_t b = *B; + tommy_uint64_t a_lo = (tommy_uint32_t)a; + tommy_uint64_t a_hi = a >> 32; + tommy_uint64_t b_lo = (tommy_uint32_t)b; + tommy_uint64_t b_hi = b >> 32; + tommy_uint64_t p0 = a_lo * b_lo; + tommy_uint64_t p1 = a_lo * b_hi; + tommy_uint64_t p2 = a_hi * b_lo; + tommy_uint64_t p3 = a_hi * b_hi; + + /* retain the carry from the middle words in the high half */ + tommy_uint64_t middle = p1 + (p0 >> 32) + (tommy_uint32_t)p2; + tommy_uint64_t low = (middle << 32) | (tommy_uint32_t)p0; + tommy_uint64_t high = p3 + (middle >> 32) + (p2 >> 32); + *A = low; + *B = high; +} #endif + +#endif + diff --git a/tommyds/tommyhashdyn.c b/tommyds/tommyhashdyn.c index 54d213f2..6775d5dd 100644 --- a/tommyds/tommyhashdyn.c +++ b/tommyds/tommyhashdyn.c @@ -4,6 +4,8 @@ #include "tommyhashdyn.h" #include "tommylist.h" +#include /* for memset */ + /******************************************************************************/ /* hashdyn */ @@ -23,10 +25,16 @@ TOMMY_API void tommy_hashdyn_done(tommy_hashdyn* hashdyn) tommy_free(hashdyn->bucket); } +TOMMY_API void tommy_hashdyn_clear(tommy_hashdyn* hashdyn) +{ + memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); + hashdyn->count = 0; +} + /** * Resize the bucket vector. */ -static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket_bit) +static void tommy_imp_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket_bit) { tommy_size_t bucket_bit = hashdyn->bucket_bit; tommy_size_t bucket_max = hashdyn->bucket_max; @@ -37,57 +45,72 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket if (!hashdyn->count) { /* empty table: allocate zeroed memory with calloc without looping */ new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); - } else if (new_bucket_bit == bucket_bit + 1) { - tommy_size_t i; - - /* grow by 1 bit: zero the two buckets inline in loop to preserve cache locality */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); - - for (i = 0; i < bucket_max; ++i) { - tommy_hashdyn_node* j; - - new_bucket[i] = 0; - new_bucket[i + bucket_max] = 0; - - j = hashdyn->bucket[i]; - while (j) { - tommy_hashdyn_node* j_next = j->next; - tommy_size_t pos = j->index & new_bucket_mask; - if (new_bucket[pos]) - tommy_list_insert_tail_not_empty(new_bucket[pos], j); - else - tommy_list_insert_first(&new_bucket[pos], j); - j = j_next; - } - } } else if (new_bucket_bit > bucket_bit) { - tommy_size_t i; + if (new_bucket_bit == bucket_bit + 1) { + /* grow by 1 bit: zero the two buckets inline in loop to preserve cache locality */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); - /* grow by multiple bits with elements: allocate zeroed memory and reinsert */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); + for (tommy_size_t i = 0; i < bucket_max; ++i) { + new_bucket[i] = 0; + new_bucket[i + bucket_max] = 0; - for (i = 0; i < bucket_max; ++i) { - tommy_hashdyn_node* j = hashdyn->bucket[i]; + tommy_hashdyn_node* j = hashdyn->bucket[i]; + if (j && !j->next) { + /* growth keeps old buckets separate, and singleton links are already valid */ + new_bucket[j->index & new_bucket_mask] = j; + continue; + } + while (j) { + tommy_hashdyn_node* j_next = j->next; + tommy_size_t pos = j->index & new_bucket_mask; + if (new_bucket[pos]) + tommy_list_insert_tail_not_empty(new_bucket[pos], j); + else + tommy_list_insert_first(&new_bucket[pos], j); + j = j_next; + } + } + } else { + /* grow by multiple bits with elements: allocate zeroed memory and reinsert */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); - while (j) { - tommy_hashdyn_node* j_next = j->next; - tommy_size_t pos = j->index & new_bucket_mask; - if (new_bucket[pos]) - tommy_list_insert_tail_not_empty(new_bucket[pos], j); - else - tommy_list_insert_first(&new_bucket[pos], j); - j = j_next; + for (tommy_size_t i = 0; i < bucket_max; ++i) { + tommy_hashdyn_node* j = hashdyn->bucket[i]; + + if (j && !j->next) { + /* growth keeps old buckets separate, and singleton links are already valid */ + new_bucket[j->index & new_bucket_mask] = j; + continue; + } + while (j) { + tommy_hashdyn_node* j_next = j->next; + tommy_size_t pos = j->index & new_bucket_mask; + if (new_bucket[pos]) + tommy_list_insert_tail_not_empty(new_bucket[pos], j); + else + tommy_list_insert_first(&new_bucket[pos], j); + j = j_next; + } } } } else { - tommy_size_t i; - - /* shrink with elements: all buckets are overwritten, no pre-zeroing needed */ + /* all buckets are overwritten, no pre-zeroing needed */ new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); - for (i = 0; i < new_bucket_max; ++i) { - new_bucket[i] = hashdyn->bucket[i]; - tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); + if (new_bucket_bit + 1 == bucket_bit) { + /* shrink by 1 bit: each new bucket joins exactly two old buckets */ + for (tommy_size_t i = 0; i < new_bucket_max; ++i) { + new_bucket[i] = hashdyn->bucket[i]; + tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); + } + } else { + /* shrink by multiple bits */ + for (tommy_size_t i = 0; i < new_bucket_max; ++i) { + /* all old buckets with the same new modulus must be concatenated. */ + new_bucket[i] = hashdyn->bucket[i]; + for (tommy_size_t j = i + new_bucket_max; j < bucket_max; j += new_bucket_max) + tommy_list_concat(&new_bucket[i], &hashdyn->bucket[j]); + } } } @@ -103,21 +126,21 @@ static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket /** * Grow. */ -tommy_inline void hashdyn_grow_step(tommy_hashdyn* hashdyn) +tommy_inline void tommy_imp_hashdyn_grow_step(tommy_hashdyn* hashdyn) { - /* grow if more than 50% full */ + /* grow if at least 50% full */ if (hashdyn->count >= hashdyn->bucket_max / 2) - tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); + tommy_imp_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); } /** * Shrink. */ -tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn) +tommy_inline void tommy_imp_hashdyn_shrink_step(tommy_hashdyn* hashdyn) { - /* shrink if less than 12.5% full */ + /* shrink if at most 12.5% full */ if (hashdyn->count <= hashdyn->bucket_max / 8 && hashdyn->bucket_bit > TOMMY_HASHDYN_BIT) - tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); + tommy_imp_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); } TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count) @@ -130,7 +153,20 @@ TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count) if (bucket_bit <= hashdyn->bucket_bit) return; - tommy_hashdyn_resize(hashdyn, bucket_bit); + tommy_imp_hashdyn_resize(hashdyn, bucket_bit); +} + +TOMMY_API void tommy_hashdyn_shrink(tommy_hashdyn* hashdyn) +{ + tommy_uint_t bucket_bit = tommy_ilog2(hashdyn->count | 1) + 2; + + if (bucket_bit < TOMMY_HASHDYN_BIT) + bucket_bit = TOMMY_HASHDYN_BIT; + + if (bucket_bit >= hashdyn->bucket_bit) + return; + + tommy_imp_hashdyn_resize(hashdyn, bucket_bit); } TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) @@ -143,7 +179,42 @@ TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* ++hashdyn->count; - hashdyn_grow_step(hashdyn); + tommy_imp_hashdyn_grow_step(hashdyn); +} + +TOMMY_API void* tommy_hashdyn_insert_unique(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) +{ + tommy_hashdyn_node** bucket = &hashdyn->bucket[hash & hashdyn->bucket_mask]; + tommy_hashdyn_node* i = *bucket; + + while (i) { + /* compare keys only for matching full hashes, as the bucket may contain different hashes */ + if (i->index == hash && cmp(cmp_arg, i->data) == 0) + return i->data; + i = i->next; + } + + /* insert before resizing, while the bucket selected for the search is still valid */ + tommy_list_insert_tail(bucket, node, data); + + node->index = hash; + + ++hashdyn->count; + + tommy_imp_hashdyn_grow_step(hashdyn); + + return data; +} + +TOMMY_API void tommy_hashdyn_rehash_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, tommy_hash_t hash) +{ + if (node->index == hash) + return; + + /* unlink using the stored hash, without invoking the resize policy. */ + tommy_list_remove_existing(&hashdyn->bucket[node->index & hashdyn->bucket_mask], node); + node->index = hash; + tommy_list_insert_tail(&hashdyn->bucket[hash & hashdyn->bucket_mask], node, node->data); } TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) @@ -154,7 +225,7 @@ TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hash --hashdyn->count; - hashdyn_shrink_step(hashdyn); + tommy_imp_hashdyn_shrink_step(hashdyn); return node->data; } @@ -171,7 +242,7 @@ TOMMY_API void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* --hashdyn->count; - hashdyn_shrink_step(hashdyn); + tommy_imp_hashdyn_shrink_step(hashdyn); return node->data; } @@ -185,9 +256,8 @@ TOMMY_API void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* { tommy_size_t bucket_max = hashdyn->bucket_max; tommy_hashdyn_node** bucket = hashdyn->bucket; - tommy_size_t pos; - for (pos = 0; pos < bucket_max; ++pos) { + for (tommy_size_t pos = 0; pos < bucket_max; ++pos) { tommy_hashdyn_node* node = bucket[pos]; while (node) { @@ -202,9 +272,8 @@ TOMMY_API void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_a { tommy_size_t bucket_max = hashdyn->bucket_max; tommy_hashdyn_node** bucket = hashdyn->bucket; - tommy_size_t pos; - for (pos = 0; pos < bucket_max; ++pos) { + for (tommy_size_t pos = 0; pos < bucket_max; ++pos) { tommy_hashdyn_node* node = bucket[pos]; while (node) { @@ -215,68 +284,180 @@ TOMMY_API void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_a } } -TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn) +TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(const tommy_hashdyn* hashdyn) { return hashdyn->bucket_max * (tommy_size_t)sizeof(hashdyn->bucket[0]) + tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node); } -/** - * \brief Transfers all elements from the dynamic hashtable into a tommy_list. - * - * Removes every element from the \p hashdyn hashtable and inserts them - * into the provided \p list (at the tail), preserving the per-bucket order - * but not guaranteeing any particular global order. - * - * After the call: - * - the hashtable is left completely empty (\c count = 0, all buckets = NULL) - * - the target list contains all the elements that were previously in the hashtable - * - * This function is useful when you need to: - * - extract all elements to process/sort them outside the hash table - * - convert the hashtable into a list for sequential iteration - * - prepare for a full clear + re-insertion with different hash/ordering - * - move ownership of the nodes to a list-based container - * - * \note The operation is O(n) where n is the number of elements. - * \note No memory allocation is performed. - * \note The relative order of elements that were in the same bucket is preserved, - * but the order among different buckets is bucket-order dependent. - * - * Typical usage pattern: - * \code - * tommy_list all_elements; - * tommy_list_init(&all_elements); - * - * // move everything out of the hashtable into the list - * tommy_hashdyn_to_list(&hashdyn, &all_elements); - * - * // now you can sort, filter, process sequentially, etc. - * tommy_list_sort(&all_elements, compare_by_value); - * \endcode - * - * \param hashdyn The dynamic hashtable to drain - * \param list The list that will receive all the elements - */ TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list) { - tommy_size_t bucket_max = hashdyn->bucket_max; - tommy_hashdyn_node** bucket = hashdyn->bucket; - tommy_size_t pos; + tommy_builder builder; + tommy_node* builder_tail = tommy_builder_init(&builder); - /* move everything to the list */ - for (pos = 0; pos < bucket_max; ++pos) { - tommy_hashdyn_node* node = bucket[pos]; - - while (node) { - tommy_node* node_next = node->next; - tommy_list_insert_tail(list, node, node->data); - node = node_next; + /* clear buckets as they are transferred to avoid a second scan */ + for (tommy_size_t pos = 0; pos < hashdyn->bucket_max; ++pos) { + tommy_node* head = hashdyn->bucket[pos]; + if (head) { + tommy_node* tail = head->prev; + builder_tail = tommy_builder_concat(builder_tail, head, tail); } + hashdyn->bucket[pos] = 0; } - /* clear all */ - memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); + tommy_list_concat_builder(list, &builder, builder_tail); hashdyn->count = 0; } +/** + * Compares hashes in least-significant-bit-first order, with 0 before 1. + * + * The first differing bit decides the order: first precedes second when + * second has a 1 there. Equal hashes compare false. + */ +tommy_inline tommy_bool_t tommy_imp_hashdyn_before(tommy_hash_t first, tommy_hash_t second) +{ + /* mark the positions where the hashes differ using XOR */ + tommy_hash_t diff = first ^ second; + + /* isolate the lowest differing bit using two's complement */ + tommy_hash_t bit = diff & -diff; + + /* a 1 in second means first has 0 there; equal hashes yield bit == 0 */ + return (second & bit) != 0; +} + +/** + * Gets the adjacent active bucket in least-significant-bit-first order. + * Returns bucket_max when there is no adjacent bucket. + * + * Traverses the virtual binary tree of buckets in bit-reversed order. + * + * For example, with 8 buckets the forward traversal order is: + * 0, 4, 2, 6, 1, 5, 3, 7 + * + * When resizing to 16 buckets: + * 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 + */ +tommy_inline tommy_size_t tommy_imp_hashdyn_step(tommy_hashdyn* hashdyn, tommy_size_t pos, tommy_bool_t reverse) +{ + tommy_size_t bit = (hashdyn->bucket_mask + 1) / 2; + + /* propagate binary carry in bit-reversed order from MSB downwards */ + while (bit && ((pos & bit) != 0) != reverse) { + pos ^= bit; + bit >>= 1; + } + + /* carry wrapped around the root of the tree: end of traversal */ + if (!bit) + return hashdyn->bucket_max; + pos ^= bit; + + return pos; +} + +/** + * Finds the first or last node starting at an active bucket. + * + * Scans buckets in least-significant-bit-first order until a non-empty bucket + * is found, then returns its extreme element (minimum for forward, maximum for reverse). + */ +tommy_inline tommy_hashdyn_node* tommy_imp_hashdyn_limit(tommy_hashdyn* hashdyn, tommy_size_t pos, tommy_bool_t reverse) +{ + while (pos != hashdyn->bucket_max) { + tommy_hashdyn_node* best = 0; + for (tommy_hashdyn_node* i = hashdyn->bucket[pos]; i; i = i->next) { + /* equal hashes retain insertion order, so the reverse limit selects the last */ + if (reverse) { + if (!best || !tommy_imp_hashdyn_before(i->index, best->index)) + best = i; + } else { + if (!best || tommy_imp_hashdyn_before(i->index, best->index)) + best = i; + } + } + if (best) + return best; + pos = tommy_imp_hashdyn_step(hashdyn, pos, reverse); + } + return 0; +} + +/** + * Finds a neighbor using the current bucket layout and a resize-independent order. + * + * First searches for the closest neighbor within the current bucket. + * If none exists, advances to the next non-empty active bucket in LSB order. + */ +tommy_inline tommy_hashdyn_node* tommy_imp_hashdyn_neighbor(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, tommy_bool_t reverse) +{ + tommy_size_t pos = node->index & hashdyn->bucket_mask; + tommy_hashdyn_node* best = 0; + tommy_bool_t passed = 0; + + /* + * Bucket links can change during grow and shrink. Compare full hashes instead + * of following those links as iteration order, so no remaining node moves + * behind the iterator. Equal hashes always stay in one bucket in insertion order. + */ + for (tommy_hashdyn_node* i = hashdyn->bucket[pos]; i; i = i->next) { + if (i == node) { + passed = 1; + } else if (i->index == node->index) { + /* for identical hashes, follow insertion order relative to the current node */ + if (reverse && !passed) + best = i; + else if (!reverse && passed) + return i; + } else if (reverse) { + if (tommy_imp_hashdyn_before(i->index, node->index)) { + /* find the closest neighbor within the bucket in LSB order */ + if (!best || !tommy_imp_hashdyn_before(i->index, best->index)) + best = i; + } + } else { + if (tommy_imp_hashdyn_before(node->index, i->index)) { + /* find the closest neighbor within the bucket in LSB order */ + if (!best || tommy_imp_hashdyn_before(i->index, best->index)) + best = i; + } + } + } + if (best) + return best; + + /* bucket exhausted: advance to the next non-empty bucket in LSB order */ + return tommy_imp_hashdyn_limit(hashdyn, tommy_imp_hashdyn_step(hashdyn, pos, reverse), reverse); +} + +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_head(tommy_hashdyn* hashdyn) +{ + /* start forward traversal at bucket 0, which has all LSB bits at zero */ + return tommy_imp_hashdyn_limit(hashdyn, 0, 0); +} + +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_tail(tommy_hashdyn* hashdyn) +{ + /* start reverse traversal at the all-ones bucket, which has the highest LSB value */ + return tommy_imp_hashdyn_limit(hashdyn, hashdyn->bucket_mask, 1); +} + +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_next(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) +{ + /* adjacent equal hashes are consecutive in insertion order */ + if (node->next && node->next->index == node->index) + return node->next; + + return tommy_imp_hashdyn_neighbor(hashdyn, node, 0); +} + +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_prev(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) +{ + /* equal hashes follow insertion order, but the bucket head's prev points to the tail */ + if (node != hashdyn->bucket[node->index & hashdyn->bucket_mask] && node->prev->index == node->index) + return node->prev; + + return tommy_imp_hashdyn_neighbor(hashdyn, node, 1); +} + diff --git a/tommyds/tommyhashdyn.h b/tommyds/tommyhashdyn.h index 9bce46be..91018fe0 100644 --- a/tommyds/tommyhashdyn.h +++ b/tommyds/tommyhashdyn.h @@ -4,18 +4,18 @@ /** \file * Dynamic chained hashtable. * - * This hashtable resizes dynamically. It starts with the minimal size of 16 buckets, it doubles - * the size when it reaches a load factor greater than 0.5 and it halves the size when the load - * factor is lower than 0.125. + * This hashtable resizes dynamically. It starts with the minimal size of 16 buckets. + * It doubles the size when it reaches a load factor greater than or equal to 0.5 + * and it halves the size when the load factor is less than or equal to 0.125. * * All the elements are reallocated in a single resize operation done inside * tommy_hashdyn_insert() or tommy_hashdyn_remove(). * - * Note that the resize operation takes approximately 100 [ms] with 1 million of elements, - * and 1 [second] with 10 millions. This could be a problem in real-time applications. + * Note that the resize operation takes approximately 100 [ms] with 1 million elements, + * and 1 [second] with 10 million elements. This could be a problem in real-time applications. * - * The resize also **fragments** the heap, as it involves allocating a double-sized table, **copying** elements, - * and deallocating the older table, **leaving** a big hole in the heap. + * The resize also fragments the heap, as it involves allocating a double-sized table, copying elements, + * and deallocating the older table, leaving a big hole in the heap. * * The ::tommy_hashlin hashtable fixes both problems. * @@ -30,36 +30,36 @@ * To insert elements in the hashtable you have to call tommy_hashdyn_insert() for * each element. * In the insertion call you have to specify the address of the node, the - * address of the object, and the hash value of the key to use. - * The address of the object is used to initialize the tommy_node::data field - * of the node, and the hash to initialize the tommy_node::key field. + * address of the element, and the hash value of the key to use. + * The address of the element is used to initialize the tommy_node::data field + * of the node, and the hash to initialize the tommy_node::index field. * * \code - * struct object { + * struct element { * int value; * // other fields * tommy_node node; * }; * - * struct object* obj = malloc(sizeof(struct object)); // creates the object + * struct element* elem = malloc(sizeof(struct element)); // creates the element * - * obj->value = ...; // initializes the object + * elem->value = ...; // initializes the element * - * tommy_hashdyn_insert(&hashdyn, &obj->node, obj, tommy_inthash_u32(obj->value)); // inserts the object + * tommy_hashdyn_insert(&hashdyn, &elem->node, elem, tommy_inthash_u32(elem->value)); // inserts the element * \endcode * - * To find an element in the hashtable you have to call tommy_hashtable_search() + * To find an element in the hashtable you have to call tommy_hashdyn_search() * providing a comparison function, its argument, and the hash of the key to search. * * \code - * int compare(const void* arg, const void* obj) + * int compare(const void* arg, const void* elem) * { - * return *(const int*)arg != ((const struct object*)obj)->value; + * return *(const int*)arg != ((const struct element*)elem)->value; * } * * int value_to_find = 1; - * struct object* obj = tommy_hashdyn_search(&hashdyn, compare, &value_to_find, tommy_inthash_u32(value_to_find)); - * if (!obj) { + * struct element* elem = tommy_hashdyn_search(&hashdyn, compare, &value_to_find, tommy_inthash_u32(value_to_find)); + * if (!elem) { * // not found * } else { * // found @@ -67,7 +67,7 @@ * \endcode * * To iterate over all the elements in the hashtable with the same key, you have to - * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL. + * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until 0. * You have also to check explicitly for the key, as the bucket may contain * different keys. * @@ -75,10 +75,10 @@ * int value_to_find = 1; * tommy_node* i = tommy_hashdyn_bucket(&hashdyn, tommy_inthash_u32(value_to_find)); * while (i) { - * struct object* obj = i->data; // gets the object pointer + * struct element* elem = i->data; // gets the element pointer * - * if (obj->value == value_to_find) { - * printf("%d\n", obj->value); // process the object + * if (elem->value == value_to_find) { + * printf("%d\n", elem->value); // process the element * } * * i = i->next; // goes to the next element @@ -90,9 +90,9 @@ * and remove. * * \code - * struct object* obj = tommy_hashdyn_remove(&hashdyn, compare, &value_to_remove, tommy_inthash_u32(value_to_remove)); - * if (obj) { - * free(obj); // frees the object allocated memory + * struct element* elem = tommy_hashdyn_remove(&hashdyn, compare, &value_to_remove, tommy_inthash_u32(value_to_remove)); + * if (elem) { + * free(elem); // frees the element allocated memory * } * \endcode * @@ -105,15 +105,60 @@ * * If you need to iterate over all the elements in the hashtable, you can use * tommy_hashdyn_foreach() or tommy_hashdyn_foreach_arg(). - * If you need a more precise control with a real iteration, you have to insert - * all the elements also in a ::tommy_list, and use the list to iterate. - * See the \ref multiindex example for more detail. + * For traversal that permits insertions and removals, use tommy_hashdyn_head() + * and tommy_hashdyn_next(), or tommy_hashdyn_tail() and tommy_hashdyn_prev(). + * Hashes are compared from the least significant bit to the most significant + * bit, with 0 before 1. Equal hashes are visited in insertion order. + * This order is independent of the number of buckets and progressive resizing. + * It differs from the bucket order used by foreach and to_list. + * + * \anchor tommy_hashdyn_iterator_validity + * Iterators are pointers to nodes currently contained in the hashtable. + * Inserting elements or removing other nodes does not invalidate them or + * change the relative iteration order of the remaining elements. + * A forward or backward traversal visits every original element that remains + * contained, even if removals cause the table to shrink or insertions grow it. + * New elements can be visited if inserted after the current position in the + * traversal direction; elements inserted before it need not be visited. + * + * Removing the pointed node invalidates its use as an iterator, even if its + * storage has not been freed. Obtain the next or previous node before removal, + * and keep that saved node contained until continuing the traversal. + * + * \code + * tommy_hashdyn_node* i = tommy_hashdyn_head(&hashdyn); + * while (i) { + * tommy_hashdyn_node* next = tommy_hashdyn_next(&hashdyn, i); + * if (should_remove(i->data)) { + * free(tommy_hashdyn_remove_existing(&hashdyn, i)); + * } + * i = next; + * } + * \endcode + * + * Changing a hash with tommy_hashdyn_rehash_existing() can change iteration + * order, so the traversal guarantee does not cover rehashing during traversal. + * After swapping containers, use the container that now contains the node. + * Clearing or deinitializing the container invalidates all its iterators. + * Concurrent traversal and modification require external synchronization. + * + * Iterator calls scan bucket contents without allocating or modifying memory. + * With a good hash function and normal dynamic resizing, a full traversal + * runs in expected O(n) time. Under the same hash distribution assumption, + * sparse tables require expected O(n + b) time, where n is the number of + * elements and b is the number of buckets. Such tables can arise after + * tommy_hashdyn_reserve(), tommy_hashdyn_clear(), or tommy_hashdyn_to_list(). + * In pathological cases with severe hash collisions, traversal can take + * O(n^2 + b) time. + * Use foreach for guaranteed O(n + b) traversal when no insertions or removals + * are needed. */ -#ifndef __TOMMYHASHDYN_H -#define __TOMMYHASHDYN_H +#ifndef TOMMY_HASHDYN_H +#define TOMMY_HASHDYN_H #include "tommyhash.h" +#include "tommylist.h" /******************************************************************************/ /* hashdyn */ @@ -126,7 +171,7 @@ /** * Hashtable node. - * This is the node that you have to include inside your objects. + * This is the node that you have to include inside your elements. */ typedef tommy_node tommy_hashdyn_node; @@ -144,6 +189,7 @@ typedef struct tommy_hashdyn_struct { /** * Initializes the hashtable. + * \param hashdyn The hashtable. */ TOMMY_API void tommy_hashdyn_init(tommy_hashdyn* hashdyn); @@ -152,30 +198,99 @@ TOMMY_API void tommy_hashdyn_init(tommy_hashdyn* hashdyn); * * You can call this function with elements still contained, * but such elements are not going to be freed by this call. + * \param hashdyn The hashtable. */ TOMMY_API void tommy_hashdyn_done(tommy_hashdyn* hashdyn); +/** + * Exchanges the complete state of two initialized hashtables, including bucket allocations. + * The hashtables must not share nodes. Nodes and elements are not accessed or modified. + * Existing node pointers remain valid and belong to the other hashtable. + * Passing the same hashtable twice has no effect. Both hashtables remain usable and deinitializable. + * \param first The first hashtable. + * \param second The second hashtable. + * \note This operation is O(1) with respect to the number of elements. + */ +tommy_inline void tommy_hashdyn_swap(tommy_hashdyn* first, tommy_hashdyn* second) +{ + tommy_hashdyn tmp = *first; + *first = *second; + *second = tmp; +} + +/** + * Removes all elements, preserving the allocated buckets. + * Call tommy_hashdyn_shrink() after clearing to restore the initial capacity. + * The hashtable remains initialized and can be reused immediately. + * Elements are not freed and nodes are not accessed or modified. + * Their links must not be used to traverse the previous contents. + * You can call this function after tommy_hashdyn_foreach() has freed the elements. + * Subsequent insertions and removals retain the normal resizing policy. + * \param hashdyn The hashtable. + * \note This operation is O(b), where b is the number of buckets. + */ +TOMMY_API void tommy_hashdyn_clear(tommy_hashdyn* hashdyn); + /** * Pre-allocates buckets for the specified number of elements. * * It ensures that the hashtable has enough buckets allocated to contain - * the specified number of elements without triggering a dynamic resize. + * the specified number of elements without triggering a dynamic resize on insertion. + * Subsequent removals retain the normal shrinking policy and may reduce + * the allocated capacity. + * \param hashdyn The hashtable. * \param count Number of elements to reserve space for. */ TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count); +/** + * Shrinks the allocated buckets to fit the current number of elements. + * The target is the smallest power of 2, at least 16, for which + * the number of elements is strictly less than half the number of buckets. + * If the current allocation is already no larger than the target, nothing is done. + * An empty hashtable is reduced to 16 buckets. + * Elements are not freed, and the tommy_node::data and tommy_node::index fields + * are left unchanged. The order of elements with the same hash is preserved. + * \param hashdyn The hashtable. + * \note This operation is O(b), where b is the previous number of buckets. + */ +TOMMY_API void tommy_hashdyn_shrink(tommy_hashdyn* hashdyn); + /** * Inserts an element in the hashtable. + * \param hashdyn The hashtable. + * \param node Pointer to the node embedded into the element to insert. + * \param data Pointer to the element to insert. + * \param hash Hash of the element to insert. */ TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash); +/** + * Inserts an element only if no equal element is already contained. + * If found, the first equal element is returned, + * and the hashtable and candidate node are left unchanged. + * Otherwise, the candidate is inserted using the normal insertion policy, + * and data is returned. + * Elements are not freed by this call. + * \param hashdyn The hashtable. + * \param node The candidate node. It must not belong to any container. + * \param data Pointer to the element to insert. + * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. + * The function should return 0 for equal elements, non-zero for different elements. + * \param cmp_arg Compare argument describing the candidate key. + * \param hash Hash of the candidate key, consistent with the comparison function. + * \return The first equal element, or data if the candidate was inserted. + */ +TOMMY_API void* tommy_hashdyn_insert_unique(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash); + /** * Searches and removes an element from the hashtable. * You have to provide a compare function and the hash of the element you want to remove. * If the element is not found, 0 is returned. * If more equal elements are present, the first one is removed. + * \param hashdyn The hashtable. * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. - * The function should return 0 for equal elements, anything other for different elements. + * The function should return 0 for equal elements, non-zero for different elements. * \param cmp_arg Compare argument passed as first argument of the compare function. * \param hash Hash of the element to find and remove. * \return The removed element, or 0 if not found. @@ -185,8 +300,9 @@ TOMMY_API void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* /** * Gets the bucket of the specified hash. * The bucket is guaranteed to contain ALL the elements with the specified hash, - * but it can contain also others. - * You can access elements in the bucket following the ::next pointer until 0. + * but it can also contain others. + * You can access elements in the bucket following the tommy_node::next pointer until 0. + * \param hashdyn The hashtable. * \param hash Hash of the element to find. * \return The head of the bucket, or 0 if empty. */ @@ -196,35 +312,123 @@ tommy_inline tommy_hashdyn_node* tommy_hashdyn_bucket(tommy_hashdyn* hashdyn, to } /** - * Searches an element in the hashtable. + * Finds a node in the hashtable. * You have to provide a compare function and the hash of the element you want to find. * If more equal elements are present, the first one is returned. + * \param hashdyn The hashtable. * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. - * The function should return 0 for equal elements, anything other for different elements. + * The function should return 0 for equal elements, non-zero for different elements. * \param cmp_arg Compare argument passed as first argument of the compare function. * \param hash Hash of the element to find. - * \return The first element found, or 0 if none. + * \return The first node found, or 0 if none. */ -tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) +tommy_inline tommy_hashdyn_node* tommy_hashdyn_find(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) { tommy_hashdyn_node* i = tommy_hashdyn_bucket(hashdyn, hash); while (i) { /* we first check if the hash matches, as in the same bucket we may have multiple hash values */ if (i->index == hash && cmp(cmp_arg, i->data) == 0) - return i->data; + return i; i = i->next; } return 0; } +/** + * Searches an element in the hashtable. + * You have to provide a compare function and the hash of the element you want to find. + * If more equal elements are present, the first one is returned. + * \param hashdyn The hashtable. + * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. + * The function should return 0 for equal elements, non-zero for different elements. + * \param cmp_arg Compare argument passed as first argument of the compare function. + * \param hash Hash of the element to find. + * \return The first element found, or 0 if none. + */ +tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) +{ + tommy_hashdyn_node* node = tommy_hashdyn_find(hashdyn, cmp, cmp_arg, hash); + + return node ? node->data : 0; +} + /** * Removes an element from the hashtable. - * You must already have the address of the element to remove. - * \return The tommy_node::data field of the node removed. + * You must already have the address of the node to remove. + * \param hashdyn The hashtable. + * \param node The node to remove. + * \return The removed element. */ TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); +/** + * Gets the first node in iteration order, or 0 if empty. + * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. + * \param hashdyn The hashtable. + * \return The first node in iteration order, or 0 if empty. + * \note This operation is expected O(1) under suitable hash distribution and + * normal dynamic resizing. In the worst case, scanning bucket contents and + * empty buckets takes O(n + b), where n is the number of elements and b is + * the number of buckets. + */ +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_head(tommy_hashdyn* hashdyn); + +/** + * Gets the last node in iteration order, or 0 if empty. + * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. + * \param hashdyn The hashtable. + * \return The last node in iteration order, or 0 if empty. + * \note This operation is expected O(1) under suitable hash distribution and + * normal dynamic resizing. In the worst case, scanning bucket contents and + * empty buckets takes O(n + b), where n is the number of elements and b is + * the number of buckets. + */ +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_tail(tommy_hashdyn* hashdyn); + +/** + * Gets the following node in iteration order, or 0 for the tail. + * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. + * \param hashdyn The hashtable. + * \param node Node currently contained in the hashtable. It must not be 0. + * \return The following node in iteration order, or 0 for the tail. + * \note This operation is expected O(1) under suitable hash distribution and + * normal dynamic resizing. In the worst case, scanning bucket contents and + * empty buckets takes O(n + b), where n is the number of elements and b is + * the number of buckets. + */ +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_next(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); + +/** + * Gets the preceding node in iteration order, or 0 for the head. + * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. + * \param hashdyn The hashtable. + * \param node Node currently contained in the hashtable. It must not be 0. + * \return The preceding node in iteration order, or 0 for the head. + * \note This operation is expected O(1) under suitable hash distribution and + * normal dynamic resizing. In the worst case, scanning bucket contents and + * empty buckets takes O(n + b), where n is the number of elements and b is + * the number of buckets. + */ +TOMMY_API tommy_hashdyn_node* tommy_hashdyn_prev(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); + +/** + * Updates the hash of an element already contained in the hashtable. + * The node must belong to this hashtable. The caller updates the element key + * and provides its new hash, without modifying tommy_node::index directly. + * If the hash is unchanged, the node and its position are left unchanged. + * Otherwise, the node is moved to the tail of the destination bucket, + * even if the old and new hashes identify the same bucket. + * The tommy_node::data field and the number of elements are left unchanged. + * No memory allocation, deallocation or resize is performed. + * Equal keys are allowed; no uniqueness check is performed. + * \param hashdyn The hashtable. + * \param node The node whose hash is updated. + * \param hash The new hash of the element. + * \note This operation is O(1). + */ +TOMMY_API void tommy_hashdyn_rehash_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, tommy_hash_t hash); + /** * Calls the specified function for each element in the hashtable. * @@ -239,66 +443,102 @@ TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hash * * ... * - * // creates an object - * struct object* obj = malloc(sizeof(struct object)); + * // creates an element + * struct element* elem = malloc(sizeof(struct element)); * * ... * * // insert it in the hashtable - * tommy_hashdyn_insert(&hashdyn, &obj->node, obj, tommy_inthash_u32(obj->value)); + * tommy_hashdyn_insert(&hashdyn, &elem->node, elem, tommy_inthash_u32(elem->value)); * * ... * - * // deallocates all the objects iterating the hashtable + * // deallocates all the elements iterating the hashtable * tommy_hashdyn_foreach(&hashdyn, free); * * // deallocates the hashtable * tommy_hashdyn_done(&hashdyn); * \endcode + * + * If the callback frees the elements, the hashtable retains dangling + * node pointers. Call tommy_hashdyn_clear() before reusing it, or + * tommy_hashdyn_done() to release its storage. + * \param hashdyn The hashtable. + * \param func Function to call with each element. */ TOMMY_API void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func); /** * Calls the specified function with an argument for each element in the hashtable. + * The iteration order and callback rules are the same as tommy_hashdyn_foreach(). + * The callback may deallocate the current element. + * Adding or removing elements from inside the callback is not allowed. + * \param hashdyn The hashtable. + * \param func Function to call with each element. + * \param arg Argument to pass to the function. */ TOMMY_API void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg); /** * Gets the number of elements. + * \param hashdyn The hashtable. + * \return The number of elements in the hashtable. + * \note This operation is O(1). */ -tommy_inline tommy_size_t tommy_hashdyn_count(tommy_hashdyn* hashdyn) +tommy_inline tommy_size_t tommy_hashdyn_count(const tommy_hashdyn* hashdyn) { return hashdyn->count; } /** - * Gets the size of allocated memory. - * It includes the size of the ::tommy_hashdyn_node of the stored elements. + * Checks whether the hashtable is empty. + * \param hashdyn The hashtable. + * \return Nonzero if the hashtable is empty, 0 otherwise. + * \note This operation is O(1). */ -TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn); +tommy_inline tommy_bool_t tommy_hashdyn_empty(const tommy_hashdyn* hashdyn) +{ + return hashdyn->count == 0; +} /** - * \brief Transfers all elements from the hashtable into a tommy_list. + * Gets the number of buckets. + * \param hashdyn The hashtable. + * \return The number of buckets. + * \note This operation is O(1). + */ +tommy_inline tommy_size_t tommy_hashdyn_bucket_count(const tommy_hashdyn* hashdyn) +{ + return hashdyn->bucket_max; +} + +/** + * Gets the size of allocated memory. + * It includes the size of the ::tommy_hashdyn_node of the stored elements. + * \param hashdyn The hashtable. + * \return Total memory allocated by the hashtable, in bytes. + */ +TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(const tommy_hashdyn* hashdyn); + +/** + * Transfers all elements from the hashtable into a tommy_list. * * Removes every element from the \p hashdyn hashtable and inserts them * into the provided \p list (at the tail), preserving the per-bucket order * but not guaranteeing any particular global order. * * After the call: - * - the hashtable is left completely empty + * - the hashtable is left empty and initialized, preserving its allocated buckets * - the target list contains all the elements that were previously in the hashtable * + * The tommy_node::data and tommy_node::index fields are left unchanged. + * * This function is useful when you need to: - * - extract all elements to process/sort them outside the hash table + * - extract all elements to process/sort them outside the hashtable * - convert the hashtable into a list for sequential iteration * - prepare for a full clear + re-insertion with different hash/ordering * - move ownership of the nodes to a list-based container * - * \note The operation is O(n) where n is the number of elements. - * \note No memory allocation is performed. - * \note The relative order of elements that were in the same bucket is preserved, - * but the order among different buckets is bucket-order dependent. - * * Typical usage pattern: * \code * tommy_list all_elements; @@ -311,8 +551,13 @@ TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn); * tommy_list_sort(&all_elements, compare_by_value); * \endcode * - * \param hashdyn The dynamic hashtable to drain - * \param list The list that will receive all the elements + * \param hashdyn The hashtable to drain. + * \param list The destination list. It must be initialized and must not share + * nodes with the hashtable. Existing elements remain at the head of the list. + * \note This operation is O(b), where b is the number of buckets. + * \note No memory allocation or deallocation is performed. + * \note The relative order of elements that were in the same bucket is preserved, + * but the order among different buckets is bucket-order dependent. */ TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list); diff --git a/tommyds/tommylist.c b/tommyds/tommylist.c index 9f7b2102..a1d4d8ef 100644 --- a/tommyds/tommylist.c +++ b/tommyds/tommylist.c @@ -7,30 +7,109 @@ /** \internal * Setup a list. */ -tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) +tommy_inline void tommy_imp_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) { head->prev = tail; tail->next = 0; *list = head; } +TOMMY_API void tommy_list_splice_before(tommy_list* list, tommy_node* reference, tommy_list* second) +{ + tommy_node* second_head = tommy_list_head(second); + + if (!second_head) + return; + + if (reference == tommy_list_head(list)) { + /* prepend */ + tommy_list_prepend(list, second); + } else { + /* splice in the middle */ + tommy_chain_splice(reference->prev, reference, second_head, second_head->prev); + } +} + +TOMMY_API void tommy_list_splice_after(tommy_list* list, tommy_node* reference, tommy_list* second) +{ + tommy_node* second_head = tommy_list_head(second); + + if (!second_head) + return; + + if (!reference->next) { + /* append */ + tommy_list_concat(list, second); + } else { + /* splice in the middle */ + tommy_chain_splice(reference, reference->next, second_head, second_head->prev); + } +} + +TOMMY_API void tommy_list_merge(tommy_list* first, tommy_list* second, tommy_compare_func* cmp) +{ + tommy_node* second_head = tommy_list_head(second); + + if (!second_head) + return; + + tommy_node* first_head = tommy_list_head(first); + + if (!first_head) { + *first = *second; + return; + } + + /* create chains from the non-empty lists */ + tommy_chain first_chain; + tommy_chain second_chain; + first_chain.head = first_head; + first_chain.tail = first_head->prev; + second_chain.head = second_head; + second_chain.tail = second_head->prev; + + tommy_chain_merge_degenerated(&first_chain, &second_chain, cmp); + + /* restore the destination list */ + tommy_imp_list_set(first, first_chain.head, first_chain.tail); +} + +TOMMY_API void tommy_list_reverse(tommy_list* list) +{ + tommy_node* head = tommy_list_head(list); + tommy_node* node = head; + tommy_node* prev = 0; + + if (!head) + return; + + while (node) { + tommy_node* next = node->next; + node->next = prev; + node->prev = next; + prev = node; + node = next; + } + + /* the old head is the new tail */ + tommy_imp_list_set(list, prev, head); +} + TOMMY_API void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp) { - tommy_chain chain; - tommy_node* head; - if (tommy_list_empty(list)) return; - head = tommy_list_head(list); + tommy_node* head = tommy_list_head(list); /* create a chain from the list */ + tommy_chain chain; chain.head = head; chain.tail = head->prev; tommy_chain_mergesort(&chain, cmp); /* restore the list */ - tommy_list_set(list, chain.head, chain.tail); + tommy_imp_list_set(list, chain.head, chain.tail); } diff --git a/tommyds/tommylist.h b/tommyds/tommylist.h index 88423358..4de562fd 100644 --- a/tommyds/tommylist.h +++ b/tommyds/tommylist.h @@ -2,21 +2,21 @@ // Copyright (C) 2010 Andrea Mazzoleni /** \file - * Double linked list for collisions into hashtables. + * Doubly linked list for collisions into hashtables. * - * This list is a **doubly** linked list mainly targeted for handling collisions - * into an hashtable, but **usable** also as a generic list. + * This list is a doubly linked list mainly targeted for handling collisions + * into a hashtable, but usable also as a generic list. * * The main feature of this list is to require only one pointer to represent the - * list, compared to a classic implementation requiring a head **and** a tail pointers. + * list, compared to a classic implementation requiring both head and tail pointers. * This reduces the memory usage in hashtables. * - * Another feature is to support the insertion at the end of the list. This **allows** to store - * collisions in a stable order. Where for stable order we mean that equal elements keep + * Another feature is to support the insertion at the end of the list. This allows + * storing collisions in a stable order, meaning that equal elements preserve * their insertion order. * * To initialize the list, you have to call tommy_list_init(), or to simply assign - * to it NULL, as an empty list is represented by the NULL value. + * to it 0, as an empty list is represented by the 0 value. * * \code * tommy_list list; @@ -27,51 +27,113 @@ * To insert elements in the list you have to call tommy_list_insert_tail() * or tommy_list_insert_head() for each element. * In the insertion call you have to specify the address of the node and the - * address of the object. - * The address of the object is used to initialize the tommy_node::data field + * address of the element. + * The address of the element is used to initialize the tommy_node::data field * of the node. * * \code - * struct object { + * struct element { * int value; * // other fields * tommy_node node; * }; * - * struct object* obj = malloc(sizeof(struct object)); // creates the object + * struct element* elem = malloc(sizeof(struct element)); // creates the element * - * obj->value = ...; // initializes the object + * elem->value = ...; // initializes the element * - * tommy_list_insert_tail(&list, &obj->node, obj); // inserts the object + * tommy_list_insert_tail(&list, &elem->node, elem); // inserts the element * \endcode * * To iterate over all the elements in the list you have to call * tommy_list_head() to get the head of the list and follow the - * tommy_node::next pointer until NULL. + * tommy_node::next pointer until 0. * * \code * tommy_node* i = tommy_list_head(&list); * while (i) { - * struct object* obj = i->data; // gets the object pointer + * struct element* elem = i->data; // gets the element pointer * - * printf("%d\n", obj->value); // process the object + * printf("%d\n", elem->value); // process the element * * i = i->next; // go to the next element * } * \endcode * + * To iterate backward, start at tommy_list_tail() and use tommy_list_prev(). + * Do not follow tommy_node::prev directly past the head, as it points to the tail. + * + * \code + * tommy_node* i = tommy_list_tail(&list); + * while (i) { + * struct element* elem = i->data; + * + * printf("%d\n", elem->value); + * + * i = tommy_list_prev(&list, i); + * } + * \endcode + * + * \anchor tommy_list_iterator_validity + * Iterators are pointers to nodes contained in the list. + * Inserting elements or removing other nodes between iterator calls does not + * invalidate pointers to nodes that remain in the list. + * Subsequent next/prev calls use the current contents and ordering; + * inserted elements may become visible and removed elements are skipped. + * + * Removing the pointed node invalidates its use as an iterator, even if + * its storage has not been freed. To continue traversal after removing + * the current node, obtain the next or previous node before removal. + * + * \code + * tommy_node* i = tommy_list_head(&list); + * while (i) { + * tommy_node* next = i->next; // obtain the next node before removal + * + * if (condition) { + * tommy_list_remove_existing(&list, i); + * free(i->data); // safe to deallocate + * } + * + * i = next; + * } + * \endcode + * + * Concurrent traversal and modification require external synchronization. + * + * To reorder existing elements, use tommy_list_move_head(), tommy_list_move_tail() + * or tommy_list_reverse(). To split a list or transfer elements between lists, + * use tommy_list_split(), tommy_list_prepend(), tommy_list_splice_before() + * and tommy_list_splice_after(). + * These operations preserve the element pointers and do not allocate memory. + * + * \code + * tommy_list second; + * tommy_list_init(&second); + * + * // transfers node and all following nodes to second + * tommy_list_split(&list, node, &second); + * + * // appends the transferred nodes back to list + * tommy_list_concat(&list, &second); + * \endcode + * + * Two lists sorted with the same comparison function can be merged with + * tommy_list_merge(). The merge is stable. + * * To destroy the list you have to remove all the elements, - * as the list is completely inplace and it doesn't allocate memory. + * as the list is completely in-place and it doesn't allocate memory. * This can be done with the tommy_list_foreach() function. * * \code - * // deallocates all the objects iterating the list + * // deallocates all the elements iterating the list * tommy_list_foreach(&list, free); + * tommy_list_clear(&list); // resets the list before reuse * \endcode */ -#ifndef __TOMMYLIST_H -#define __TOMMYLIST_H +#ifndef TOMMY_LIST_H +#define TOMMY_LIST_H #include "tommytypes.h" @@ -83,18 +145,72 @@ */ typedef tommy_node* tommy_list; +/** + * Temporary list builder with an embedded sentinel. + * Initialize with tommy_builder_init() and finish with tommy_list_concat_builder(). + * Keep the builder at the same address until finalization, as node links + * refer to its sentinel. + */ +typedef struct tommy_builder_struct { + tommy_node dummy; /**< Sentinel excluded from the resulting list. */ +} tommy_builder; + +/** + * Initializes an empty builder in O(1). + * \param builder Builder to initialize in its final location. + * \return Initial tail, pointing to the builder's sentinel. + */ +tommy_inline tommy_node* tommy_builder_init(tommy_builder* builder) +{ + builder->dummy.next = 0; + return &builder->dummy; +} + +/** + * Appends a non-empty chain to a builder in O(1), preserving element order. + * \param tail Current tail, initially returned by tommy_builder_init(). + * \param head Head of the non-empty source chain. + * \param last Tail of the source chain. For a single node, pass head again. + * The chains must not share nodes. Inner links must be valid; + * head's prev and last's next may be undefined. + * \return Updated tail to pass to the next append or tommy_list_concat_builder(). + */ +tommy_inline tommy_node* tommy_builder_concat(tommy_node* tail, tommy_node* head, tommy_node* last) +{ + head->prev = tail; + tail->next = head; + return last; +} + /** * Initializes the list. - * The list is completely inplace, so it doesn't need to be deinitialized. + * The list is completely in-place, so it doesn't need to be deinitialized. + * \param list The list. */ tommy_inline void tommy_list_init(tommy_list* list) { *list = 0; } +/** + * Removes all elements. + * The list remains initialized and can be reused immediately. + * Elements are not freed and nodes are not accessed or modified. + * Their links must not be used to traverse the previous contents. + * You can call this function after tommy_list_foreach() has freed the elements. + * \param list The list. + * \note This operation is O(1). + */ +tommy_inline void tommy_list_clear(tommy_list* list) +{ + *list = 0; +} + /** * Gets the head of the list. - * \return The head node. For empty lists 0 is returned. + * \param list The list. + * \return The head node, or 0 if the list is empty. + * \note This operation is O(1). */ tommy_inline tommy_node* tommy_list_head(tommy_list* list) { @@ -103,7 +219,9 @@ tommy_inline tommy_node* tommy_list_head(tommy_list* list) /** * Gets the tail of the list. - * \return The tail node. For empty lists 0 is returned. + * \param list The list. + * \return The tail node, or 0 if the list is empty. + * \note This operation is O(1). */ tommy_inline tommy_node* tommy_list_tail(tommy_list* list) { @@ -115,9 +233,27 @@ tommy_inline tommy_node* tommy_list_tail(tommy_list* list) return head->prev; } +/** + * Gets the previous node in the list. + * Unlike tommy_node::prev, this function returns 0 at the head of the list, + * allowing backward iteration starting from tommy_list_tail(). + * See \ref tommy_list_iterator_validity for iterator validity across insertions and removals. + * \param list The list. + * \param node The current node. The node must be in the list. + * \return The previous node, or 0 if the current node is the head. + * \note This operation is O(1). + */ +tommy_inline tommy_node* tommy_list_prev(tommy_list* list, tommy_node* node) +{ + if (node == tommy_list_head(list)) + return 0; + + return node->prev; +} + /** \internal * Creates a new list with a single element. - * \param list The list to initialize. + * \param list The list. * \param node The node to insert. */ tommy_inline void tommy_list_insert_first(tommy_list* list, tommy_node* node) @@ -134,7 +270,7 @@ tommy_inline void tommy_list_insert_first(tommy_list* list, tommy_node* node) /** \internal * Inserts an element at the head of a non-empty list. * The element is inserted at the head of the list. The list cannot be empty. - * \param list The list. The list cannot be empty. + * \param list The list. * \param node The node to insert. */ tommy_inline void tommy_list_insert_head_not_empty(tommy_list* list, tommy_node* node) @@ -171,8 +307,8 @@ tommy_inline void tommy_list_insert_tail_not_empty(tommy_node* head, tommy_node* /** * Inserts an element at the head of a list. * \param list The list. - * \param node The node to insert. - * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + * \param node Pointer to the node embedded into the element to insert. + * \param data Pointer to the element to insert. */ tommy_inline void tommy_list_insert_head(tommy_list* list, tommy_node* node, void* data) { @@ -189,8 +325,8 @@ tommy_inline void tommy_list_insert_head(tommy_list* list, tommy_node* node, voi /** * Inserts an element at the tail of a list. * \param list The list. - * \param node The node to insert. - * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + * \param node Pointer to the node embedded into the element to insert. + * \param data Pointer to the element to insert. */ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, void* data) { @@ -208,8 +344,8 @@ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, voi * Inserts an element before the specified reference node. * \param list The list. * \param reference The reference node. The new node will be inserted before this node. The reference node must be in the list. - * \param node The node to insert. - * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + * \param node Pointer to the node embedded into the element to insert. + * \param data Pointer to the element to insert. */ tommy_inline void tommy_list_insert_before(tommy_list* list, tommy_node* reference, tommy_node* node, void* data) { @@ -235,8 +371,8 @@ tommy_inline void tommy_list_insert_before(tommy_list* list, tommy_node* referen * Inserts an element after the specified reference node. * \param list The list. * \param reference The reference node. The new node will be inserted after this node. The reference node must be in the list. - * \param node The node to insert. - * \param data The object containing the node. It's used to set the tommy_node::data field of the node. + * \param node Pointer to the node embedded into the element to insert. + * \param data Pointer to the element to insert. */ tommy_inline void tommy_list_insert_after(tommy_list* list, tommy_node* reference, tommy_node* node, void* data) { @@ -260,12 +396,12 @@ tommy_inline void tommy_list_insert_after(tommy_list* list, tommy_node* referenc /** * Removes an element from the list. - * You must already have the address of the element to remove. - * \note The node content is left unchanged, including the tommy_node::next - * and tommy_node::prev fields that still contain pointers **in** the list. + * You must already have the address of the node to remove. * \param list The list. - * \param node The node to remove. The node must be in the list. - * \return The tommy_node::data field of the node removed. + * \param node The node to remove. + * \return The removed element. + * \note The node content is left unchanged, including the tommy_node::next + * and tommy_node::prev fields that still contain pointers in the list. */ tommy_inline void* tommy_list_remove_existing(tommy_list* list, tommy_node* node) { @@ -287,32 +423,167 @@ tommy_inline void* tommy_list_remove_existing(tommy_list* list, tommy_node* node } /** - * Concats two lists. - * The second list is concatenated at the first list. + * Removes the element at the head of the list. + * \param list The list. + * \return The removed element, or 0 if the list is empty. + */ +tommy_inline void* tommy_list_remove_head(tommy_list* list) +{ + tommy_node* node = tommy_list_head(list); + + if (!node) + return 0; + + /* the new head must retain the tail pointer in the "circular" prev list */ + if (node->next) + node->next->prev = node->prev; + + *list = node->next; + + return node->data; +} + +/** + * Removes the element at the tail of the list. + * \param list The list. + * \return The removed element, or 0 if the list is empty. + */ +tommy_inline void* tommy_list_remove_tail(tommy_list* list) +{ + tommy_node* head = tommy_list_head(list); + + if (!head) + return 0; + + tommy_node* node = head->prev; + + /* a single node is also the head, and its links must remain unchanged */ + if (node == head) { + *list = 0; + } else { + head->prev = node->prev; + node->prev->next = 0; + } + + return node->data; +} + +/** + * Moves an existing node to the head of the list. + * If the node is already at the head, nothing is done. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param list The list. + * \param node The node to move. The node must be in the list. + */ +tommy_inline void tommy_list_move_head(tommy_list* list, tommy_node* node) +{ + if (node == tommy_list_head(list)) + return; + + tommy_list_remove_existing(list, node); + tommy_list_insert_head_not_empty(list, node); +} + +/** + * Moves an existing node to the tail of the list. + * If the node is already at the tail, nothing is done. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param list The list. + * \param node The node to move. The node must be in the list. + */ +tommy_inline void tommy_list_move_tail(tommy_list* list, tommy_node* node) +{ + if (node->next == 0) + return; + + tommy_list_remove_existing(list, node); + tommy_list_insert_tail_not_empty(tommy_list_head(list), node); +} + +/** + * Splits a list before the specified node. + * The list retains the nodes preceding the specified node. The second list + * receives the specified node and all following nodes. The order of the elements is preserved. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param list The list to split. + * \param node The first node to transfer. The node must be in the list, + * or 0 to leave list unchanged and set second to 0. + * Specifying the head transfers the whole list. + * \param second Output list receiving node and all following nodes. + * It must be distinct from list and need not be initialized. + * Its previous value is overwritten. Any previously referenced nodes + * are left unchanged and are not deallocated. + * \note This operation is O(1). + */ +tommy_inline void tommy_list_split(tommy_list* list, tommy_node* node, tommy_list* second) +{ + tommy_node* head = tommy_list_head(list); + tommy_node* tail; + tommy_node* before; + + if (!node) { + *second = 0; + return; + } + + if (node == head) { + *second = *list; + *list = 0; + return; + } + + tail = head->prev; + before = node->prev; + + /* terminate the retained list */ + head->prev = before; + before->next = 0; + + /* create the list of transferred nodes */ + node->prev = tail; + *second = node; +} + +/** + * Exchanges the contents of two lists. + * The lists must not share nodes. Nodes and elements are not accessed or modified. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * Passing the same list twice has no effect. + * \param first The first list. + * \param second The second list. + * \note This operation is O(1). + */ +tommy_inline void tommy_list_swap(tommy_list* first, tommy_list* second) +{ + tommy_list tmp = *first; + *first = *second; + *second = tmp; +} + +/** + * Concatenates two lists. + * The second list is concatenated to the first list. * \param first The first list. * \param second The second list. After this call the list content is undefined, * and you should not use it anymore. + * \note This operation is O(1). */ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second) { - tommy_node* first_head; - tommy_node* first_tail; - tommy_node* second_head; - /* if the second is empty, nothing to do */ - second_head = tommy_list_head(second); + tommy_node* second_head = tommy_list_head(second); if (second_head == 0) return; /* if the first is empty, copy the second */ - first_head = tommy_list_head(first); + tommy_node* first_head = tommy_list_head(first); if (first_head == 0) { *first = *second; return; } /* tail of the first list */ - first_tail = first_head->prev; + tommy_node* first_tail = first_head->prev; /* set the "circular" prev list */ first_head->prev = second_head->prev; @@ -322,32 +593,136 @@ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second) first_tail->next = second_head; } +/** + * Appends a builder's chain to a list, preserving element order and restoring list links. + * \param list Destination list, possibly empty. It must not share nodes with the builder. + * \param builder Initialized builder, possibly empty. Its contents are consumed; + * reinitialize it before reuse. + * \param tail Current tail returned by the builder helpers, ignored when empty. + * \note This operation is O(1). An empty builder has no effect. + */ +tommy_inline void tommy_list_concat_builder(tommy_list* list, const tommy_builder* builder, tommy_node* tail) +{ + tommy_node* head = builder->dummy.next; + + if (!head) + return; + + tommy_node* first = *list; + + /* replace the sentinel link so no list node refers to the temporary builder */ + if (first) { + tommy_node* first_tail = first->prev; + first_tail->next = head; + head->prev = first_tail; + first->prev = tail; + } else { + *list = head; + head->prev = tail; + } + + tail->next = 0; +} + +/** + * Prepends a list to another list. + * The second list is prepended to the first list. + * \param first The first list. + * \param second The second list. After this call the list content is undefined, + * and you should not use it anymore. + * \note This operation is O(1). + */ +tommy_inline void tommy_list_prepend(tommy_list* first, tommy_list* second) +{ + tommy_list_concat(second, first); + *first = *second; +} + +/** + * Transfers all elements of a list before the specified reference node. + * The order of the transferred elements is preserved. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param list The destination list. + * \param reference The node before which to insert. It must not be 0 and must be in the destination list. + * \param second The source list. After this call the list content is undefined, + * and you should not use it anymore. + * The lists must be distinct and must not share nodes. + * \note This operation is O(1). An empty source list has no effect. + */ +TOMMY_API void tommy_list_splice_before(tommy_list* list, tommy_node* reference, tommy_list* second); + +/** + * Transfers all elements of a list after the specified reference node. + * The order of the transferred elements is preserved. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param list The destination list. + * \param reference The node after which to insert. It must not be 0 and must be in the destination list. + * \param second The source list. After this call the list content is undefined, + * and you should not use it anymore. + * The lists must be distinct and must not share nodes. + * \note This operation is O(1). An empty source list has no effect. + */ +TOMMY_API void tommy_list_splice_after(tommy_list* list, tommy_node* reference, tommy_list* second); + +/** + * Merges two sorted lists. + * Both lists must be sorted using the same comparison function. + * The merge is stable: equal elements from the first list precede those from + * the second list, and the relative order within each list is preserved. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param first The destination list, receiving the sorted result. + * \param second The source list. After this call the list content is undefined, + * and you should not use it anymore. + * The lists must be distinct and must not share nodes. + * \param cmp Compare function called with two elements. + * The function should return <0 if the first element is less than the second, + * ==0 if equal, and >0 if greater. It must not modify the lists. + * \note This operation is O(n+m), or O(1) when one list is empty or the ranges + * can be concatenated directly. It uses O(1) auxiliary space. + */ +TOMMY_API void tommy_list_merge(tommy_list* first, tommy_list* second, tommy_compare_func* cmp); + +/** + * Reverses the order of the elements in a list. + * Empty lists and lists containing one element are left unchanged. + * The tommy_node::data and tommy_node::index fields are left unchanged. + * \param list The list. + * \note This operation is O(n) and uses O(1) auxiliary space. + */ +TOMMY_API void tommy_list_reverse(tommy_list* list); + /** * Sorts a list. - * It's a stable merge sort with O(N*log(N)) worst complexity. - * It's faster on degenerated cases like partially ordered lists. + * It's a stable merge sort with O(n log n) worst complexity. + * It's faster on degenerate cases like partially ordered lists. + * \param list The list. * \param cmp Compare function called with two elements. * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greater. + * \note This operation is O(n log n) in the worst case. */ TOMMY_API void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp); /** - * Checks if empty. - * \return If the list is empty. + * Checks whether the list is empty. + * \param list The list. + * \return Nonzero if the list is empty, 0 otherwise. + * \note This operation is O(1). */ -tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list) +tommy_inline tommy_bool_t tommy_list_empty(const tommy_list* list) { - return tommy_list_head(list) == 0; + return *list == 0; } /** * Gets the number of elements. + * \param list The list. + * \return The number of elements in the list. * \note This operation is O(n). */ -tommy_inline tommy_size_t tommy_list_count(tommy_list* list) +tommy_inline tommy_size_t tommy_list_count(const tommy_list* list) { tommy_size_t count = 0; - tommy_node* i = tommy_list_head(list); + const tommy_node* i = *list; while (i) { ++count; @@ -371,19 +746,26 @@ tommy_inline tommy_size_t tommy_list_count(tommy_list* list) * * ... * - * // creates an object - * struct object* obj = malloc(sizeof(struct object)); + * // creates an element + * struct element* elem = malloc(sizeof(struct element)); * * ... * * // insert it in the list - * tommy_list_insert_tail(&list, &obj->node, obj); + * tommy_list_insert_tail(&list, &elem->node, elem); * * ... * - * // deallocates all the objects iterating the list + * // deallocates all the elements iterating the list * tommy_list_foreach(&list, free); + * tommy_list_clear(&list); // resets the list before reuse * \endcode + * + * If the callback frees the elements, the list retains dangling node + * pointers. Call tommy_list_clear() or tommy_list_init() before reusing it. + * The list has no storage to release. + * \param list The list. + * \param func Function to call with each element. */ tommy_inline void tommy_list_foreach(tommy_list* list, tommy_foreach_func* func) { @@ -398,6 +780,12 @@ tommy_inline void tommy_list_foreach(tommy_list* list, tommy_foreach_func* func) /** * Calls the specified function with an argument for each element in the list. + * The iteration order and callback rules are the same as tommy_list_foreach(). + * The callback may deallocate the current element. + * Adding or removing elements from inside the callback is not allowed. + * \param list The list. + * \param func Function to call with each element. + * \param arg Argument to pass to the function. */ tommy_inline void tommy_list_foreach_arg(tommy_list* list, tommy_foreach_arg_func* func, void* arg) { @@ -411,3 +799,4 @@ tommy_inline void tommy_list_foreach_arg(tommy_list* list, tommy_foreach_arg_fun } #endif + diff --git a/tommyds/tommytree.c b/tommyds/tommytree.c index 7f3fcd19..069401be 100644 --- a/tommyds/tommytree.c +++ b/tommyds/tommytree.c @@ -14,7 +14,7 @@ #define TOMMY_TREE_BALANCE_LEFT ((tommy_size_t)1) #define TOMMY_TREE_BALANCE_RIGHT ((tommy_size_t)2) -tommy_inline int tommy_tree_balance_get(tommy_tree_node* node) +tommy_inline int tommy_imp_tree_balance_get(tommy_tree_node* node) { tommy_size_t balance = node->index & TOMMY_TREE_BALANCE_MASK; @@ -25,7 +25,7 @@ tommy_inline int tommy_tree_balance_get(tommy_tree_node* node) return 0; } -tommy_inline void tommy_tree_balance_set(tommy_tree_node* node, int balance) +tommy_inline void tommy_imp_tree_balance_set(tommy_tree_node* node, int balance) { tommy_size_t value = 0; @@ -37,12 +37,12 @@ tommy_inline void tommy_tree_balance_set(tommy_tree_node* node, int balance) node->index = (node->index & ~TOMMY_TREE_BALANCE_MASK) | value; } -tommy_inline void tommy_tree_parent_set(tommy_tree_node* node, tommy_tree_node* parent) +tommy_inline void tommy_imp_tree_parent_set(tommy_tree_node* node, tommy_tree_node* parent) { node->index = (tommy_size_t)(tommy_uintptr_t)parent | (node->index & TOMMY_TREE_BALANCE_MASK); } -tommy_inline void tommy_tree_replace(tommy_tree* tree, tommy_tree_node* root, tommy_tree_node* node) +tommy_inline void tommy_imp_tree_replace(tommy_tree* tree, tommy_tree_node* root, tommy_tree_node* node) { tommy_tree_node* parent = tommy_tree_parent(root); @@ -55,45 +55,45 @@ tommy_inline void tommy_tree_replace(tommy_tree* tree, tommy_tree_node* root, to } if (node) - tommy_tree_parent_set(node, parent); + tommy_imp_tree_parent_set(node, parent); } -tommy_inline tommy_tree_node* tommy_tree_rotate_left(tommy_tree* tree, tommy_tree_node* root) +tommy_inline tommy_tree_node* tommy_imp_tree_rotate_left(tommy_tree* tree, tommy_tree_node* root) { tommy_tree_node* next = root->next; - tommy_tree_replace(tree, root, next); + tommy_imp_tree_replace(tree, root, next); root->next = next->prev; if (root->next) - tommy_tree_parent_set(root->next, root); + tommy_imp_tree_parent_set(root->next, root); next->prev = root; - tommy_tree_parent_set(root, next); + tommy_imp_tree_parent_set(root, next); return next; } -tommy_inline tommy_tree_node* tommy_tree_rotate_right(tommy_tree* tree, tommy_tree_node* root) +tommy_inline tommy_tree_node* tommy_imp_tree_rotate_right(tommy_tree* tree, tommy_tree_node* root) { tommy_tree_node* prev = root->prev; - tommy_tree_replace(tree, root, prev); + tommy_imp_tree_replace(tree, root, prev); root->prev = prev->next; if (root->prev) - tommy_tree_parent_set(root->prev, root); + tommy_imp_tree_parent_set(root->prev, root); prev->next = root; - tommy_tree_parent_set(root, prev); + tommy_imp_tree_parent_set(root, prev); return prev; } -tommy_inline void tommy_tree_insert_balance(tommy_tree* tree, tommy_tree_node* node) +tommy_inline void tommy_imp_tree_insert_balance(tommy_tree* tree, tommy_tree_node* node) { tommy_tree_node* parent = tommy_tree_parent(node); while (parent) { - int balance = tommy_tree_balance_get(parent); + int balance = tommy_imp_tree_balance_get(parent); if (node == parent->prev) --balance; @@ -102,12 +102,12 @@ tommy_inline void tommy_tree_insert_balance(tommy_tree* tree, tommy_tree_node* n } if (balance == 0) { - tommy_tree_balance_set(parent, balance); + tommy_imp_tree_balance_set(parent, balance); return; } if (balance >= -1 && balance <= 1) { - tommy_tree_balance_set(parent, balance); + tommy_imp_tree_balance_set(parent, balance); node = parent; parent = tommy_tree_parent(parent); continue; @@ -115,39 +115,39 @@ tommy_inline void tommy_tree_insert_balance(tommy_tree* tree, tommy_tree_node* n if (balance == -2) { tommy_tree_node* left = parent->prev; - int left_balance = tommy_tree_balance_get(left); + int left_balance = tommy_imp_tree_balance_get(left); if (left_balance < 0) { - tommy_tree_rotate_right(tree, parent); - tommy_tree_balance_set(parent, 0); - tommy_tree_balance_set(left, 0); + tommy_imp_tree_rotate_right(tree, parent); + tommy_imp_tree_balance_set(parent, 0); + tommy_imp_tree_balance_set(left, 0); } else { tommy_tree_node* middle = left->next; - int middle_balance = tommy_tree_balance_get(middle); + int middle_balance = tommy_imp_tree_balance_get(middle); - tommy_tree_rotate_left(tree, left); - tommy_tree_rotate_right(tree, parent); - tommy_tree_balance_set(parent, middle_balance < 0 ? 1 : 0); - tommy_tree_balance_set(left, middle_balance > 0 ? -1 : 0); - tommy_tree_balance_set(middle, 0); + tommy_imp_tree_rotate_left(tree, left); + tommy_imp_tree_rotate_right(tree, parent); + tommy_imp_tree_balance_set(parent, middle_balance < 0 ? 1 : 0); + tommy_imp_tree_balance_set(left, middle_balance > 0 ? -1 : 0); + tommy_imp_tree_balance_set(middle, 0); } } else { tommy_tree_node* right = parent->next; - int right_balance = tommy_tree_balance_get(right); + int right_balance = tommy_imp_tree_balance_get(right); if (right_balance > 0) { - tommy_tree_rotate_left(tree, parent); - tommy_tree_balance_set(parent, 0); - tommy_tree_balance_set(right, 0); + tommy_imp_tree_rotate_left(tree, parent); + tommy_imp_tree_balance_set(parent, 0); + tommy_imp_tree_balance_set(right, 0); } else { tommy_tree_node* middle = right->prev; - int middle_balance = tommy_tree_balance_get(middle); + int middle_balance = tommy_imp_tree_balance_get(middle); - tommy_tree_rotate_right(tree, right); - tommy_tree_rotate_left(tree, parent); - tommy_tree_balance_set(parent, middle_balance > 0 ? -1 : 0); - tommy_tree_balance_set(right, middle_balance < 0 ? 1 : 0); - tommy_tree_balance_set(middle, 0); + tommy_imp_tree_rotate_right(tree, right); + tommy_imp_tree_rotate_left(tree, parent); + tommy_imp_tree_balance_set(parent, middle_balance > 0 ? -1 : 0); + tommy_imp_tree_balance_set(right, middle_balance < 0 ? 1 : 0); + tommy_imp_tree_balance_set(middle, 0); } } @@ -155,41 +155,59 @@ tommy_inline void tommy_tree_insert_balance(tommy_tree* tree, tommy_tree_node* n } } -TOMMY_API void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) +tommy_inline void* tommy_imp_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data, tommy_bool_t unique) { tommy_tree_node* parent = 0; + tommy_tree_node* existing = 0; tommy_tree_node** link = &tree->root; + /* cache the comparator to avoid reloading it after each comparison */ + tommy_compare_func* cmp_func = tree->cmp; + + while (*link) { + /* comparators must not change the tree, so keep the node across the call */ + parent = *link; + int cmp = cmp_func(data, parent->data); + + if (cmp < 0) + link = &parent->prev; + else if (cmp == 0 && unique) { + existing = parent; + link = &parent->prev; + } else { + /* placing equal keys after existing ones preserves insertion order through rotations */ + link = &parent->next; + } + } + + if (existing) + return existing->data; node->data = data; node->prev = 0; node->next = 0; - node->index = 0; - - while (*link) { - int cmp = tree->cmp(data, (*link)->data); - - parent = *link; - if (cmp < 0) - link = &parent->prev; - else if (cmp > 0) - link = &parent->next; - else - return parent->data; - } - node->index = (tommy_size_t)(tommy_uintptr_t)parent; *link = node; ++tree->count; - tommy_tree_insert_balance(tree, node); + tommy_imp_tree_insert_balance(tree, node); return node->data; } -tommy_inline void tommy_tree_remove_balance(tommy_tree* tree, tommy_tree_node* node, tommy_bool_t left_shrunk) +TOMMY_API void tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) +{ + tommy_imp_tree_insert(tree, node, data, 0); +} + +TOMMY_API void* tommy_tree_insert_unique(tommy_tree* tree, tommy_tree_node* node, void* data) +{ + return tommy_imp_tree_insert(tree, node, data, 1); +} + +tommy_inline void tommy_imp_tree_remove_balance(tommy_tree* tree, tommy_tree_node* node, tommy_bool_t left_shrunk) { while (node) { - int balance = tommy_tree_balance_get(node); + int balance = tommy_imp_tree_balance_get(node); if (left_shrunk) ++balance; @@ -197,15 +215,13 @@ tommy_inline void tommy_tree_remove_balance(tommy_tree* tree, tommy_tree_node* n --balance; if (balance == -1 || balance == 1) { - tommy_tree_balance_set(node, balance); + tommy_imp_tree_balance_set(node, balance); return; } if (balance == 0) { - tommy_tree_node* parent; - - tommy_tree_balance_set(node, balance); - parent = tommy_tree_parent(node); + tommy_imp_tree_balance_set(node, balance); + tommy_tree_node* parent = tommy_tree_parent(node); if (!parent) return; left_shrunk = node == parent->prev; @@ -215,27 +231,27 @@ tommy_inline void tommy_tree_remove_balance(tommy_tree* tree, tommy_tree_node* n if (balance == -2) { tommy_tree_node* left = node->prev; - int left_balance = tommy_tree_balance_get(left); + int left_balance = tommy_imp_tree_balance_get(left); tommy_tree_node* root; if (left_balance <= 0) { - root = tommy_tree_rotate_right(tree, node); + root = tommy_imp_tree_rotate_right(tree, node); if (left_balance == 0) { - tommy_tree_balance_set(node, -1); - tommy_tree_balance_set(left, 1); + tommy_imp_tree_balance_set(node, -1); + tommy_imp_tree_balance_set(left, 1); return; } - tommy_tree_balance_set(node, 0); - tommy_tree_balance_set(left, 0); + tommy_imp_tree_balance_set(node, 0); + tommy_imp_tree_balance_set(left, 0); } else { tommy_tree_node* middle = left->next; - int middle_balance = tommy_tree_balance_get(middle); + int middle_balance = tommy_imp_tree_balance_get(middle); - tommy_tree_rotate_left(tree, left); - root = tommy_tree_rotate_right(tree, node); - tommy_tree_balance_set(node, middle_balance < 0 ? 1 : 0); - tommy_tree_balance_set(left, middle_balance > 0 ? -1 : 0); - tommy_tree_balance_set(middle, 0); + tommy_imp_tree_rotate_left(tree, left); + root = tommy_imp_tree_rotate_right(tree, node); + tommy_imp_tree_balance_set(node, middle_balance < 0 ? 1 : 0); + tommy_imp_tree_balance_set(left, middle_balance > 0 ? -1 : 0); + tommy_imp_tree_balance_set(middle, 0); } node = tommy_tree_parent(root); @@ -247,27 +263,27 @@ tommy_inline void tommy_tree_remove_balance(tommy_tree* tree, tommy_tree_node* n { tommy_tree_node* right = node->next; - int right_balance = tommy_tree_balance_get(right); + int right_balance = tommy_imp_tree_balance_get(right); tommy_tree_node* root; if (right_balance >= 0) { - root = tommy_tree_rotate_left(tree, node); + root = tommy_imp_tree_rotate_left(tree, node); if (right_balance == 0) { - tommy_tree_balance_set(node, 1); - tommy_tree_balance_set(right, -1); + tommy_imp_tree_balance_set(node, 1); + tommy_imp_tree_balance_set(right, -1); return; } - tommy_tree_balance_set(node, 0); - tommy_tree_balance_set(right, 0); + tommy_imp_tree_balance_set(node, 0); + tommy_imp_tree_balance_set(right, 0); } else { tommy_tree_node* middle = right->prev; - int middle_balance = tommy_tree_balance_get(middle); + int middle_balance = tommy_imp_tree_balance_get(middle); - tommy_tree_rotate_right(tree, right); - root = tommy_tree_rotate_left(tree, node); - tommy_tree_balance_set(node, middle_balance > 0 ? -1 : 0); - tommy_tree_balance_set(right, middle_balance < 0 ? 1 : 0); - tommy_tree_balance_set(middle, 0); + tommy_imp_tree_rotate_right(tree, right); + root = tommy_imp_tree_rotate_left(tree, node); + tommy_imp_tree_balance_set(node, middle_balance > 0 ? -1 : 0); + tommy_imp_tree_balance_set(right, middle_balance < 0 ? 1 : 0); + tommy_imp_tree_balance_set(middle, 0); } node = tommy_tree_parent(root); @@ -289,7 +305,7 @@ TOMMY_API void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* no parent = tommy_tree_parent(node); left_shrunk = parent && node == parent->prev; - tommy_tree_replace(tree, node, child); + tommy_imp_tree_replace(tree, node, child); } else { tommy_tree_node* next = node->next; @@ -298,67 +314,106 @@ TOMMY_API void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* no parent = tommy_tree_parent(next); if (parent == node) { - tommy_tree_replace(tree, node, next); + tommy_imp_tree_replace(tree, node, next); next->prev = node->prev; - tommy_tree_parent_set(next->prev, next); - tommy_tree_balance_set(next, tommy_tree_balance_get(node)); + tommy_imp_tree_parent_set(next->prev, next); + tommy_imp_tree_balance_set(next, tommy_imp_tree_balance_get(node)); parent = next; left_shrunk = 0; } else { parent->prev = next->next; if (parent->prev) - tommy_tree_parent_set(parent->prev, parent); + tommy_imp_tree_parent_set(parent->prev, parent); - tommy_tree_replace(tree, node, next); + tommy_imp_tree_replace(tree, node, next); next->prev = node->prev; next->next = node->next; - tommy_tree_parent_set(next->prev, next); - tommy_tree_parent_set(next->next, next); - tommy_tree_balance_set(next, tommy_tree_balance_get(node)); + tommy_imp_tree_parent_set(next->prev, next); + tommy_imp_tree_parent_set(next->next, next); + tommy_imp_tree_balance_set(next, tommy_imp_tree_balance_get(node)); left_shrunk = 1; } } --tree->count; if (parent) - tommy_tree_remove_balance(tree, parent, left_shrunk); + tommy_imp_tree_remove_balance(tree, parent, left_shrunk); return data; } -TOMMY_API void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func) +TOMMY_API void tommy_tree_to_list(tommy_tree* tree, tommy_list* list) { - while (root) { - tommy_tree_node* next; + /* AVL height is less than twice the bit width of its node count */ + tommy_tree_node* stack[2 * TOMMY_SIZE_BIT]; + tommy_size_t depth = 0; + tommy_tree_node* node = tree->root; + tommy_builder builder; + tommy_node* builder_tail = tommy_builder_init(&builder); - tommy_tree_foreach_node(root->prev, func); + while (node || depth) { + while (node) { + stack[depth] = node; + ++depth; + node = node->prev; + } - /* make a copy in case func is free() */ - next = root->next; + node = stack[--depth]; + /* chain appending overwrites child links, so keep the right subtree reachable */ + tommy_tree_node* next = node->next; + builder_tail = tommy_builder_concat(builder_tail, node, node); + node = next; + } - func(root->data); + tommy_list_concat_builder(list, &builder, builder_tail); + tommy_tree_clear(tree); +} - root = next; +TOMMY_API void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) +{ + /* AVL height is less than twice the bit width of its node count */ + tommy_tree_node* stack[2 * TOMMY_SIZE_BIT]; + tommy_size_t depth = 0; + tommy_tree_node* node = tree->root; + + while (node || depth) { + while (node) { + stack[depth] = node; + ++depth; + node = node->prev; + } + + node = stack[--depth]; + /* save the next subtree before the callback can free this node */ + tommy_tree_node* next = node->next; + func(node->data); + node = next; } } -TOMMY_API void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg) +TOMMY_API void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) { - while (root) { - tommy_tree_node* next; + /* AVL height is less than twice the bit width of its node count */ + tommy_tree_node* stack[2 * TOMMY_SIZE_BIT]; + tommy_size_t depth = 0; + tommy_tree_node* node = tree->root; - tommy_tree_foreach_arg_node(root->prev, func, arg); + while (node || depth) { + while (node) { + stack[depth] = node; + ++depth; + node = node->prev; + } - /* make a copy in case func is free() */ - next = root->next; - - func(arg, root->data); - - root = next; + node = stack[--depth]; + /* save the next subtree before the callback can free this node */ + tommy_tree_node* next = node->next; + func(arg, node->data); + node = next; } } -TOMMY_API tommy_size_t tommy_tree_memory_usage(tommy_tree* tree) +TOMMY_API tommy_size_t tommy_tree_memory_usage(const tommy_tree* tree) { return tommy_tree_count(tree) * sizeof(tommy_tree_node); } diff --git a/tommyds/tommytree.h b/tommyds/tommytree.h index 9bae8611..f2ff9f99 100644 --- a/tommyds/tommytree.h +++ b/tommyds/tommytree.h @@ -7,7 +7,8 @@ * This tree is a standard AVL tree implementation that stores elements in the * order defined by the comparison function. * - * As a difference from other tommy containers, duplicate elements cannot be inserted. + * Duplicate elements are allowed. They are visited in insertion order. + * Use tommy_tree_insert_unique() to reject duplicate keys. * * To initialize a tree you have to call tommy_tree_init() specifying a comparison * function that will define the order in the tree. @@ -21,31 +22,31 @@ * To insert elements in the tree you have to call tommy_tree_insert() for * each element. * In the insertion call you have to specify the address of the node and the - * address of the object. - * The address of the object is used to initialize the tommy_node::data field + * address of the element. + * The address of the element is used to initialize the tommy_node::data field * of the node. * * \code - * struct object { + * struct element { * int value; * // other fields * tommy_tree_node node; * }; * - * struct object* obj = malloc(sizeof(struct object)); // creates the object + * struct element* elem = malloc(sizeof(struct element)); // creates the element * - * obj->value = ...; // initializes the object + * elem->value = ...; // initializes the element * - * tommy_tree_insert(&tree, &obj->node, obj); // inserts the object + * tommy_tree_insert(&tree, &elem->node, elem); // inserts the element * \endcode * - * To find an element in the tree you have to call tommy_tree_search() providing - * the key to search. + * To find the first element with a given key in tree order, call + * tommy_tree_search() providing the key to search. * * \code - * struct object value_to_find = { 1 }; - * struct object* obj = tommy_tree_search(&tree, &value_to_find); - * if (!obj) { + * struct element value_to_find = { 1 }; + * struct element* elem = tommy_tree_search(&tree, &value_to_find); + * if (!elem) { * // not found * } else { * // found @@ -59,22 +60,36 @@ * \code * tommy_tree_node* i = tommy_tree_head(&tree); * while (i) { - * struct object* obj = i->data; // gets the object pointer + * struct element* elem = i->data; // gets the element pointer * - * printf("%d\n", obj->value); // process the object + * printf("%d\n", elem->value); // process the element * * i = tommy_tree_next(i); // go to the next element * } * \endcode * - * To remove an element from the tree you have to call tommy_tree_remove() - * providing the key to search and remove. + * \anchor tommy_tree_iterator_validity + * Iterators are pointers to nodes contained in the tree. + * Inserting elements or removing other nodes between iterator calls does not + * invalidate pointers to nodes that remain in the tree. + * Subsequent next/prev calls use the current contents and ordering; + * inserted elements may become visible and removed elements are skipped. + * + * Removing the pointed node invalidates its use as an iterator, even if + * its storage has not been freed. To continue traversal after removing + * the current node, obtain the next or previous node before removal. + * + * Keys and fields used for ordering must not be modified while contained. + * Concurrent traversal and modification require external synchronization. + * + * To remove the first element with a given key in tree order, call + * tommy_tree_remove() providing the key to search and remove. * * \code - * struct object value_to_remove = { 1 }; - * struct object* obj = tommy_tree_remove(&tree, &value_to_remove); - * if (obj) { - * free(obj); // frees the object allocated memory + * struct element value_to_remove = { 1 }; + * struct element* elem = tommy_tree_remove(&tree, &value_to_remove); + * if (elem) { + * free(elem); // frees the element allocated memory * } * \endcode * @@ -82,18 +97,18 @@ * address using tommy_tree_remove_existing(). * * \code - * struct object* obj = ...; - * tommy_tree_remove_existing(&tree, &obj->node); - * free(obj); + * struct element* elem = ...; + * tommy_tree_remove_existing(&tree, &elem->node); + * free(elem); * \endcode * * To remove the smallest or greatest element you can use tommy_tree_remove_head() * or tommy_tree_remove_tail(). * * \code - * struct object* obj = tommy_tree_remove_head(&tree); - * if (obj) { - * free(obj); + * struct element* elem = tommy_tree_remove_head(&tree); + * if (elem) { + * free(elem); * } * \endcode * @@ -102,22 +117,23 @@ * This can be done with the tommy_tree_foreach() function. * * \code - * // deallocates all the objects iterating the tree + * // deallocates all the elements iterating the tree * tommy_tree_foreach(&tree, free); * \endcode */ -#ifndef __TOMMYTREE_H -#define __TOMMYTREE_H +#ifndef TOMMY_TREE_H +#define TOMMY_TREE_H #include "tommytypes.h" +#include "tommylist.h" /******************************************************************************/ /* tree */ /** * Tree node. - * This is the node that you have to include inside your objects. + * This is the node that you have to include inside your elements. * The node must preserve its natural alignment because the tree stores the * balance factor in the two least significant bits of the parent pointer. */ @@ -135,6 +151,7 @@ typedef struct tommy_tree_struct { /** * Initializes the tree. + * \param tree The tree. * \param cmp The comparison function that defines the order in the tree. */ tommy_inline void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) @@ -145,40 +162,81 @@ tommy_inline void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) } /** - * Inserts an element in the tree. - * If the element is already present, it's not inserted again. - * Check the return value to identify if the element was already present or not. - * You have to provide the pointer of the node embedded into the object and - * the pointer to the object. - * \param node Pointer to the node embedded into the object to insert. - * \param data Pointer to the object to insert. - * \return The element in the tree. Either the already existing one, or the one just inserted. + * Exchanges the contents of two initialized trees, including their comparison functions. + * Each comparison function moves with its root to preserve the tree order. + * The trees must not share nodes. Nodes are not accessed or modified. + * Passing the same tree twice has no effect. + * \param first The first tree. + * \param second The second tree. + * \note This operation is O(1). */ -TOMMY_API void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); +tommy_inline void tommy_tree_swap(tommy_tree* first, tommy_tree* second) +{ + tommy_tree tmp = *first; + *first = *second; + *second = tmp; +} + +/** + * Removes all elements, preserving the comparison function. + * The tree remains initialized and can be reused immediately. + * Elements are not freed and nodes are not accessed or modified. + * Their links must not be used to traverse the previous contents. + * You can call this function after tommy_tree_foreach() has freed the elements. + * \param tree The tree. + * \note This operation is O(1). + */ +tommy_inline void tommy_tree_clear(tommy_tree* tree) +{ + tree->root = 0; + tree->count = 0; +} + +/** + * Inserts an element in the tree, allowing duplicate keys. + * Equal elements are visited in insertion order. + * You have to provide the pointer of the node embedded into the element and + * the pointer to the element. + * \param tree The tree. + * \param node Pointer to the node embedded into the element to insert. + * \param data Pointer to the element to insert. + */ +TOMMY_API void tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); + +/** + * Inserts an element only if no equal element is already contained. + * If found, the first equal element in tree order is returned and the + * candidate node is left unchanged. Otherwise the candidate is inserted + * and data is returned. + * \param tree The tree. + * \param node The candidate node. It must not belong to any container. + * \param data Pointer to the element to insert. + * \return The first equal element, or data if the candidate was inserted. + */ +TOMMY_API void* tommy_tree_insert_unique(tommy_tree* tree, tommy_tree_node* node, void* data); + +/** + * Transfers all elements from the tree to the tail of a list in tree order. + * Existing list elements remain before the transferred elements. The tree is + * left empty and initialized with its original comparison function. + * The list must be initialized and must not share nodes with the tree. + * Elements are not freed, and the node data and index fields are left unchanged. + * The index field no longer represents a valid tree parent after the transfer. + * \param tree The tree to drain. + * \param list The destination list. + * \note This operation is O(n), using O(log n) stack space. + */ +TOMMY_API void tommy_tree_to_list(tommy_tree* tree, tommy_list* list); /** * Removes an element from the tree. - * You must already have the address of the element to remove. - * \return The tommy_node::data field of the node removed. + * You must already have the address of the node to remove. + * \param tree The tree. + * \param node The node to remove. + * \return The removed element. */ TOMMY_API void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node); - -/** \internal - * Calls the specified function for each element in the tree. - * \param root Root node of the tree. - * \param func Function to call. - */ -TOMMY_API void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func); - -/** \internal - * Calls the specified function with an argument for each element in the tree. - * \param root Root node of the tree. - * \param func Function to call. - * \param arg Custom argument for the function. - */ -TOMMY_API void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg); - /** * Gets the parent of the specified node. * \param node Node contained in the tree. @@ -191,13 +249,13 @@ tommy_inline tommy_tree_node* tommy_tree_parent(tommy_tree_node* node) /** * Gets the node following the specified one in tree order. + * See \ref tommy_tree_iterator_validity for iterator validity across insertions and removals. * \param node Node contained in the tree. - * \return The following node. For the tail node 0 is returned. + * \return The following node, or 0 for the tail node. + * \note This operation is O(log n), and amortized O(1) during full traversal. */ tommy_inline tommy_tree_node* tommy_tree_next(tommy_tree_node* node) { - tommy_tree_node* parent; - if (node->next) { node = node->next; while (node->prev) @@ -205,7 +263,7 @@ tommy_inline tommy_tree_node* tommy_tree_next(tommy_tree_node* node) return node; } - parent = tommy_tree_parent(node); + tommy_tree_node* parent = tommy_tree_parent(node); while (parent && node == parent->next) { node = parent; parent = tommy_tree_parent(parent); @@ -216,13 +274,13 @@ tommy_inline tommy_tree_node* tommy_tree_next(tommy_tree_node* node) /** * Gets the node preceding the specified one in tree order. + * See \ref tommy_tree_iterator_validity for iterator validity across insertions and removals. * \param node Node contained in the tree. - * \return The preceding node. For the head node 0 is returned. + * \return The preceding node, or 0 for the head node. + * \note This operation is O(log n), and amortized O(1) during full traversal. */ tommy_inline tommy_tree_node* tommy_tree_prev(tommy_tree_node* node) { - tommy_tree_node* parent; - if (node->prev) { node = node->prev; while (node->next) @@ -230,7 +288,7 @@ tommy_inline tommy_tree_node* tommy_tree_prev(tommy_tree_node* node) return node; } - parent = tommy_tree_parent(node); + tommy_tree_node* parent = tommy_tree_parent(node); while (parent && node == parent->prev) { node = parent; parent = tommy_tree_parent(parent); @@ -240,8 +298,10 @@ tommy_inline tommy_tree_node* tommy_tree_prev(tommy_tree_node* node) } /** - * Gets the head (smallest) element in the tree. - * \return The head node. For empty trees 0 is returned. + * Gets the head (smallest) node in the tree. + * \param tree The tree. + * \return The head node, or 0 if the tree is empty. + * \note This operation is O(log n). */ tommy_inline tommy_tree_node* tommy_tree_head(tommy_tree* tree) { @@ -257,8 +317,10 @@ tommy_inline tommy_tree_node* tommy_tree_head(tommy_tree* tree) } /** - * Gets the tail (greatest) element in the tree. - * \return The tail node. For empty trees 0 is returned. + * Gets the tail (greatest) node in the tree. + * \param tree The tree. + * \return The tail node, or 0 if the tree is empty. + * \note This operation is O(log n). */ tommy_inline tommy_tree_node* tommy_tree_tail(tommy_tree* tree) { @@ -276,7 +338,8 @@ tommy_inline tommy_tree_node* tommy_tree_tail(tommy_tree* tree) /** * Removes and returns the head (smallest) element. * If the tree is empty, 0 is returned. - * \return The removed element, or 0 if empty. + * \param tree The tree. + * \return The removed element, or 0 if the tree is empty. */ tommy_inline void* tommy_tree_remove_head(tommy_tree* tree) { @@ -291,7 +354,8 @@ tommy_inline void* tommy_tree_remove_head(tommy_tree* tree) /** * Removes and returns the tail (greatest) element. * If the tree is empty, 0 is returned. - * \return The removed element, or 0 if empty. + * \param tree The tree. + * \return The removed element, or 0 if the tree is empty. */ tommy_inline void* tommy_tree_remove_tail(tommy_tree* tree) { @@ -303,44 +367,53 @@ tommy_inline void* tommy_tree_remove_tail(tommy_tree* tree) return tommy_tree_remove_existing(tree, node); } -/** \internal - * Searches a node in the tree. - * If the element is not found, 0 is returned. - * \param cmp Comparison function. - * \param node Node to start the search from. - * \param data Element used for comparison. - * \return The first node found, or 0 if none. +/** + * Finds the first equal node in the tree with a specific comparison function. + * + * Like tommy_tree_find() but you can specify a different comparison function. + * Note that this function must define a suborder of the original one. + * The first equal node in tree order is returned. + * + * The cmp_arg argument will be the first argument of the comparison function, + * and it can be of a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first equal node in tree order, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_search_node(tommy_compare_func* cmp, tommy_tree_node* node, void* data) +tommy_inline tommy_tree_node* tommy_tree_find_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) { + tommy_tree_node* node = tree->root; + tommy_tree_node* candidate = 0; + while (node) { - int c = cmp(data, node->data); + /* comparators must not change the tree, so retain the data across the call */ + void* data = node->data; + int c = cmp(cmp_arg, data); if (c < 0) node = node->prev; else if (c > 0) node = node->next; - else - return node; + else { + candidate = node; + node = node->prev; + } } - return 0; + return candidate; } /** - * Searches an element in the tree. + * Finds the first equal node in the tree. * If the element is not found, 0 is returned. + * \param tree The tree. * \param data Element used for comparison. - * \return The first element found, or 0 if none. + * \return The first equal node in tree order, or 0 if none. */ -tommy_inline void* tommy_tree_search(tommy_tree* tree, void* data) +tommy_inline tommy_tree_node* tommy_tree_find(tommy_tree* tree, const void* data) { - tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); - - if (!node) - return 0; - - return node->data; + return tommy_tree_find_compare(tree, tree->cmp, data); } /** @@ -348,42 +421,65 @@ tommy_inline void* tommy_tree_search(tommy_tree* tree, void* data) * * Like tommy_tree_search() but you can specify a different comparison function. * Note that this function must define a suborder of the original one. + * The first equal element in tree order is returned. * - * The ::data argument will be the first argument of the comparison function, - * and it can be of a different type than the objects in the tree. + * The cmp_arg argument will be the first argument of the comparison function, + * and it can be of a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first equal element in tree order, or 0 if none. */ -tommy_inline void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) +tommy_inline void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) { - tommy_tree_node* node = tommy_tree_search_node(cmp, tree->root, cmp_arg); + tommy_tree_node* node = tommy_tree_find_compare(tree, cmp, cmp_arg); - if (!node) - return 0; - - return node->data; + return node ? node->data : 0; } -/** \internal - * Searches a node in the tree with key greater or equal than the specified one. - * If no such element exists, 0 is returned. - * \param cmp Comparison function. - * \param node Node to start the search from. +/** + * Searches an element in the tree. + * If the element is not found, 0 is returned. + * \param tree The tree. * \param data Element used for comparison. - * \return The first node found, or 0 if none. + * \return The first equal element in tree order, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_search_greater_equal_node(tommy_compare_func* cmp, tommy_tree_node* node, void* data) +tommy_inline void* tommy_tree_search(tommy_tree* tree, const void* data) { + tommy_tree_node* node = tommy_tree_find(tree, data); + + return node ? node->data : 0; +} + +/** + * Finds a node in the tree with key greater or equal than the specified one with a specific comparison function. + * + * Like tommy_tree_find_greater_equal() but you can specify a different comparison function. + * Note that this function must define a suborder of the original one. + * The first qualifying node in tree order is returned. + * + * The cmp_arg argument will be the first argument of the comparison function, + * and it can be of a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first node in tree order with key greater or equal, or 0 if none. + */ +tommy_inline tommy_tree_node* tommy_tree_find_greater_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* node = tree->root; tommy_tree_node* candidate = 0; while (node) { - int c = cmp(data, node->data); + /* comparators must not change the tree, so retain the data across the call */ + void* data = node->data; + int c = cmp(cmp_arg, data); - if (c < 0) { + if (c <= 0) { candidate = node; node = node->prev; - } else if (c > 0) { - node = node->next; } else { - return node; + node = node->next; } } @@ -391,19 +487,15 @@ tommy_inline tommy_tree_node* tommy_tree_search_greater_equal_node(tommy_compare } /** - * Searches an element in the tree with key greater or equal than the specified one. + * Finds a node in the tree with key greater or equal than the specified one. * If no such element exists, 0 is returned. + * \param tree The tree. * \param data Element used for comparison. - * \return The first element found, or 0 if none. + * \return The first node in tree order with key greater or equal, or 0 if none. */ -tommy_inline void* tommy_tree_search_greater_equal(tommy_tree* tree, void* data) +tommy_inline tommy_tree_node* tommy_tree_find_greater_equal(tommy_tree* tree, const void* data) { - tommy_tree_node* node = tommy_tree_search_greater_equal_node(tree->cmp, tree->root, data); - - if (!node) - return 0; - - return node->data; + return tommy_tree_find_greater_equal_compare(tree, tree->cmp, data); } /** @@ -411,42 +503,60 @@ tommy_inline void* tommy_tree_search_greater_equal(tommy_tree* tree, void* data) * * Like tommy_tree_search_greater_equal() but you can specify a different comparison function. * Note that this function must define a suborder of the original one. + * The first qualifying element in tree order is returned. * - * The ::data argument will be the first argument of the comparison function, - * and it can be of a different type than the objects in the tree. + * The cmp_arg argument will be the first argument of the comparison function, + * and it can be of a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first element in tree order with key greater or equal, or 0 if none. */ -tommy_inline void* tommy_tree_search_greater_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) +tommy_inline void* tommy_tree_search_greater_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) { - tommy_tree_node* node = tommy_tree_search_greater_equal_node(cmp, tree->root, cmp_arg); + tommy_tree_node* node = tommy_tree_find_greater_equal_compare(tree, cmp, cmp_arg); - if (!node) - return 0; - - return node->data; + return node ? node->data : 0; } -/** \internal - * Searches a node in the tree with key less or equal than the specified one. +/** + * Searches an element in the tree with key greater or equal than the specified one. * If no such element exists, 0 is returned. - * \param cmp Comparison function. - * \param node Node to start the search from. + * \param tree The tree. * \param data Element used for comparison. - * \return The first node found, or 0 if none. + * \return The first element in tree order with key greater or equal, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_search_less_equal_node(tommy_compare_func* cmp, tommy_tree_node* node, void* data) +tommy_inline void* tommy_tree_search_greater_equal(tommy_tree* tree, const void* data) { + tommy_tree_node* node = tommy_tree_find_greater_equal(tree, data); + + return node ? node->data : 0; +} + +/** + * Finds the first node in tree order with key strictly greater than the specified one using a specific comparison function. + * The function must define a suborder of the tree comparison function. + * Equivalent elements are excluded from the result. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first strictly greater node, or 0 if none. + */ +tommy_inline tommy_tree_node* tommy_tree_find_greater_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* node = tree->root; tommy_tree_node* candidate = 0; while (node) { - int c = cmp(data, node->data); + /* comparators must not change the tree, so retain the data across the call */ + void* data = node->data; + int c = cmp(cmp_arg, data); if (c < 0) { - node = node->prev; - } else if (c > 0) { candidate = node; - node = node->next; + node = node->prev; } else { - return node; + node = node->next; } } @@ -454,19 +564,92 @@ tommy_inline tommy_tree_node* tommy_tree_search_less_equal_node(tommy_compare_fu } /** - * Searches an element in the tree with key less or equal than the specified one. + * Finds the first node in tree order with key strictly greater than the specified one. * If no such element exists, 0 is returned. + * \param tree The tree. * \param data Element used for comparison. - * \return The first element found, or 0 if none. + * \return The first strictly greater node, or 0 if none. */ -tommy_inline void* tommy_tree_search_less_equal(tommy_tree* tree, void* data) +tommy_inline tommy_tree_node* tommy_tree_find_greater(tommy_tree* tree, const void* data) { - tommy_tree_node* node = tommy_tree_search_less_equal_node(tree->cmp, tree->root, data); + return tommy_tree_find_greater_compare(tree, tree->cmp, data); +} - if (!node) - return 0; +/** + * Searches the first element in tree order with key strictly greater than the specified one using a specific comparison function. + * The function must define a suborder of the tree comparison function. + * Equivalent elements are excluded from the result. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first strictly greater element, or 0 if none. + */ +tommy_inline void* tommy_tree_search_greater_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* node = tommy_tree_find_greater_compare(tree, cmp, cmp_arg); - return node->data; + return node ? node->data : 0; +} + +/** + * Searches the first element in tree order with key strictly greater than the specified one. + * If no such element exists, 0 is returned. + * \param tree The tree. + * \param data Element used for comparison. + * \return The first strictly greater element, or 0 if none. + */ +tommy_inline void* tommy_tree_search_greater(tommy_tree* tree, const void* data) +{ + tommy_tree_node* node = tommy_tree_find_greater(tree, data); + + return node ? node->data : 0; +} + +/** + * Finds a node in the tree with key less or equal than the specified one with a specific comparison function. + * + * Like tommy_tree_find_less_equal() but you can specify a different comparison function. + * Note that this function must define a suborder of the original one. + * The last qualifying node in tree order is returned. + * + * The cmp_arg argument will be the first argument of the comparison function, + * and it can be of a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The last node in tree order with key less or equal, or 0 if none. + */ +tommy_inline tommy_tree_node* tommy_tree_find_less_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* node = tree->root; + tommy_tree_node* candidate = 0; + + while (node) { + /* comparators must not change the tree, so retain the data across the call */ + void* data = node->data; + int c = cmp(cmp_arg, data); + + if (c < 0) { + node = node->prev; + } else { + candidate = node; + node = node->next; + } + } + + return candidate; +} + +/** + * Finds a node in the tree with key less or equal than the specified one. + * If no such element exists, 0 is returned. + * \param tree The tree. + * \param data Element used for comparison. + * \return The last node in tree order with key less or equal, or 0 if none. + */ +tommy_inline tommy_tree_node* tommy_tree_find_less_equal(tommy_tree* tree, const void* data) +{ + return tommy_tree_find_less_equal_compare(tree, tree->cmp, data); } /** @@ -474,34 +657,138 @@ tommy_inline void* tommy_tree_search_less_equal(tommy_tree* tree, void* data) * * Like tommy_tree_search_less_equal() but you can specify a different comparison function. * Note that this function must define a suborder of the original one. + * The last qualifying element in tree order is returned. * - * The ::data argument will be the first argument of the comparison function, - * and it can be of a different type than the objects in the tree. + * The cmp_arg argument will be the first argument of the comparison function, + * and it can be of a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The last element in tree order with key less or equal, or 0 if none. */ -tommy_inline void* tommy_tree_search_less_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) +tommy_inline void* tommy_tree_search_less_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) { - tommy_tree_node* node = tommy_tree_search_less_equal_node(cmp, tree->root, cmp_arg); + tommy_tree_node* node = tommy_tree_find_less_equal_compare(tree, cmp, cmp_arg); - if (!node) - return 0; - - return node->data; + return node ? node->data : 0; } /** - * Searches and removes an element. - * If the element is not found, 0 is returned. + * Searches an element in the tree with key less or equal than the specified one. + * If no such element exists, 0 is returned. + * \param tree The tree. * \param data Element used for comparison. - * \return The removed element, or 0 if not found. + * \return The last element in tree order with key less or equal, or 0 if none. */ -tommy_inline void* tommy_tree_remove(tommy_tree* tree, void* data) +tommy_inline void* tommy_tree_search_less_equal(tommy_tree* tree, const void* data) { - tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); + tommy_tree_node* node = tommy_tree_find_less_equal(tree, data); - if (!node) + return node ? node->data : 0; +} + +/** + * Finds the last node in tree order with key strictly less than the specified one using a specific comparison function. + * The function must define a suborder of the tree comparison function. + * Equivalent elements are excluded from the result. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The last strictly less node, or 0 if none. + */ +tommy_inline tommy_tree_node* tommy_tree_find_less_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* node = tree->root; + tommy_tree_node* candidate = 0; + + while (node) { + /* comparators must not change the tree, so retain the data across the call */ + void* data = node->data; + int c = cmp(cmp_arg, data); + + if (c > 0) { + candidate = node; + node = node->next; + } else { + node = node->prev; + } + } + + return candidate; +} + +/** + * Finds the last node in tree order with key strictly less than the specified one. + * If no such element exists, 0 is returned. + * \param tree The tree. + * \param data Element used for comparison. + * \return The last strictly less node, or 0 if none. + */ +tommy_inline tommy_tree_node* tommy_tree_find_less(tommy_tree* tree, const void* data) +{ + return tommy_tree_find_less_compare(tree, tree->cmp, data); +} + +/** + * Searches the last element in tree order with key strictly less than the specified one using a specific comparison function. + * The function must define a suborder of the tree comparison function. + * Equivalent elements are excluded from the result. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The last strictly less element, or 0 if none. + */ +tommy_inline void* tommy_tree_search_less_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* node = tommy_tree_find_less_compare(tree, cmp, cmp_arg); + + return node ? node->data : 0; +} + +/** + * Searches the last element in tree order with key strictly less than the specified one. + * If no such element exists, 0 is returned. + * \param tree The tree. + * \param data Element used for comparison. + * \return The last strictly less element, or 0 if none. + */ +tommy_inline void* tommy_tree_search_less(tommy_tree* tree, const void* data) +{ + tommy_tree_node* node = tommy_tree_find_less(tree, data); + + return node ? node->data : 0; +} + +/** + * Searches and removes an element from the tree with a specific comparison function. + * The comparison function must define a suborder of the original one. + * The cmp_arg argument is passed as the first argument of the comparison + * function and may have a different type than the elements in the tree. + * \param tree The tree. + * \param cmp Comparison function called with cmp_arg and the element in the tree. + * \param cmp_arg Search key, which may have a different type from tree elements. + * \return The first equal element in tree order, or 0 if not found. + */ +tommy_inline void* tommy_tree_remove_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +{ + tommy_tree_node* candidate = tommy_tree_find_compare(tree, cmp, cmp_arg); + + if (!candidate) return 0; - return tommy_tree_remove_existing(tree, node); + return tommy_tree_remove_existing(tree, candidate); +} + +/** + * Searches and removes an element from the tree. + * If the element is not found, 0 is returned. + * \param tree The tree. + * \param data Element used for comparison. + * \return The first equal element in tree order, or 0 if not found. + */ +tommy_inline void* tommy_tree_remove(tommy_tree* tree, const void* data) +{ + return tommy_tree_remove_compare(tree, tree->cmp, data); } /** @@ -520,46 +807,58 @@ tommy_inline void* tommy_tree_remove(tommy_tree* tree, void* data) * * ... * - * // creates an object - * struct object* obj = malloc(sizeof(struct object)); + * // creates an element + * struct element* elem = malloc(sizeof(struct element)); * * ... * * // insert it in the tree - * tommy_tree_insert(&tree, &obj->node, obj); + * tommy_tree_insert(&tree, &elem->node, elem); * * ... * - * // deallocates all the objects iterating the tree + * // deallocates all the elements iterating the tree * tommy_tree_foreach(&tree, free); + * tommy_tree_clear(&tree); // resets the tree before reuse * \endcode + * + * If the callback frees the elements, the tree retains dangling node + * pointers. Call tommy_tree_clear() or tommy_tree_init() before reusing it. + * The tree has no storage to release. + * \param tree The tree. + * \param func Function to call with each element. */ -tommy_inline void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) -{ - tommy_tree_foreach_node(tree->root, func); -} +TOMMY_API void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func); /** * Calls the specified function with an argument for each element in the tree. + * The iteration order and callback rules are the same as tommy_tree_foreach(). + * The callback may deallocate the current element. + * Adding or removing elements from inside the callback is not allowed. + * \param tree The tree. + * \param func Function to call with each element. + * \param arg Argument to pass to the function. */ -tommy_inline void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) -{ - tommy_tree_foreach_arg_node(tree->root, func, arg); -} +TOMMY_API void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg); /** - * Checks if empty. - * \return If the tree is empty. + * Checks whether the tree is empty. + * \param tree The tree. + * \return Nonzero if the tree is empty, 0 otherwise. + * \note This operation is O(1). */ -tommy_inline tommy_bool_t tommy_tree_empty(tommy_tree* tree) +tommy_inline tommy_bool_t tommy_tree_empty(const tommy_tree* tree) { return tree->root == 0; } /** * Gets the number of elements. + * \param tree The tree. + * \return The number of elements in the tree. + * \note This operation is O(1). */ -tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree) +tommy_inline tommy_size_t tommy_tree_count(const tommy_tree* tree) { return tree->count; } @@ -567,8 +866,10 @@ tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree) /** * Gets the size of allocated memory. * It includes the size of the ::tommy_tree_node of the stored elements. + * \param tree The tree. + * \return Total memory allocated by the tree, in bytes. */ -TOMMY_API tommy_size_t tommy_tree_memory_usage(tommy_tree* tree); +TOMMY_API tommy_size_t tommy_tree_memory_usage(const tommy_tree* tree); #endif diff --git a/tommyds/tommytypes.h b/tommyds/tommytypes.h index a63894a7..d2dd85ec 100644 --- a/tommyds/tommytypes.h +++ b/tommyds/tommytypes.h @@ -5,8 +5,8 @@ * Generic types. */ -#ifndef __TOMMYTYPES_H -#define __TOMMYTYPES_H +#ifndef TOMMY_TYPES_H +#define TOMMY_TYPES_H /******************************************************************************/ /* types */ @@ -14,15 +14,18 @@ #include #ifdef _MSC_VER +#include typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */ typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */ typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */ #ifdef _WIN64 #define TOMMY_SIZE_BIT 64 +#define TOMMY_SIZE_MAX _UI64_MAX typedef unsigned _int64 tommy_size_t; /**< Generic size_t type. */ typedef _int64 tommy_ssize_t; /**< Generic ssize_t type. */ #else #define TOMMY_SIZE_BIT 32 +#define TOMMY_SIZE_MAX UINT_MAX typedef unsigned tommy_size_t; /**< Generic size_t type. */ typedef int tommy_ssize_t; /**< Generic ssize_t type. */ #endif @@ -33,10 +36,12 @@ typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */ typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */ #if SIZE_MAX == UINT64_MAX #define TOMMY_SIZE_BIT 64 +#define TOMMY_SIZE_MAX SIZE_MAX typedef uint64_t tommy_size_t; /**< Generic size_t type. */ typedef int64_t tommy_ssize_t; /**< Generic ssize_t type. */ #elif SIZE_MAX == UINT32_MAX #define TOMMY_SIZE_BIT 32 +#define TOMMY_SIZE_MAX SIZE_MAX typedef uint32_t tommy_size_t; /**< Generic size_t type. */ typedef int32_t tommy_ssize_t; /**< Generic ssize_t type. */ #else @@ -51,7 +56,7 @@ typedef int tommy_bool_t; /**< Generic boolean type. */ * Generic unsigned integer type. * * It has no specific size, as it is used to store only small values. - * To make the code more efficient, a full 32 bit integer is used. + * To make the code more efficient, a full 32-bit integer is used. */ typedef tommy_uint32_t tommy_uint_t; @@ -80,7 +85,57 @@ typedef tommy_uint32_t tommy_uint_t; /** * Generic malloc(), calloc(), realloc() and free() functions. - * Redefine them to what you need. By default they map to the C malloc(), calloc(), realloc() and free(). + * + * Redefine them to what you need. By default they map to the C malloc(), + * calloc(), realloc() and free(). + * + * \par Infallible Allocation Invariant + * Tommy assumes that dynamic memory allocation never fails and that these + * functions always return a valid, non-zero pointer. The library does not + * check for allocation failures and does not return error codes for + * out-of-memory (OOM) conditions. + * + * \par Rationale + * This choice reflects the memory model of modern general-purpose operating + * systems: + * - **Virtual Memory Overcommit:** On modern systems (such as Linux with default + * overcommit heuristics), malloc() reserves virtual address space and almost + * never returns 0 on 64-bit systems. Physical memory is committed on first write, + * meaning actual exhaustion triggers the kernel OOM Killer (SIGKILL) rather + * than an allocation failure. Checking for null returns in userspace often + * provides a false sense of security. + * - **Swap Thrashing:** In systems backed by swap, severe memory pressure causes + * destructive disk thrashing, degrading throughput by orders of magnitude and + * rendering the process unusable long before an allocation call fails. + * + * \par Advantages + * - **Zero Error-Path Overhead:** No branch predictions, conditional jumps, or + * status checks inside critical insertion and resizing loops, maximizing CPU + * cache efficiency and execution speed. + * - **Clean and Lean API:** Functions (such as insertion and resizing) can remain + * simple (often returning void or direct element pointers) without cluttering + * client code with error-propagation boilerplate. + * - **Deterministic Consistency:** Avoids complex transactional rollback logic + * otherwise required if an allocation failed halfway through dynamic table + * growth or node migration. + * - **Industry Alignment:** Follows the established design of high-performance + * systems libraries (e.g., GLib's g_malloc, Git's xmalloc, Redis's zmalloc, + * and default C++ operator new). + * + * \par Trade-offs and Limitations + * - **Embedded and Bare-Metal Systems:** Systems without an MMU/virtual memory + * (e.g., RTOS, microcontrollers) or strict overcommit disabled (vm.overcommit_memory=2) + * do return 0 on heap exhaustion. + * - **Multi-Tenant Services:** Infrastructure daemons (e.g., database engines) that + * must isolate a single client's excessive memory request without crashing the + * entire process cannot rely on unchecked allocations. + * + * \par Custom Interception + * If your environment requires custom handling (such as memory tracking, structured + * aborts, or logging), redefine tommy_malloc, tommy_calloc, tommy_realloc, and + * tommy_free before including Tommy headers (or via compiler flags). + * Custom allocators must terminate the process (e.g., abort()) or initiate an + * unwinding non-local exit (e.g., longjmp or C++ exception) rather than returning 0. */ #if !defined(tommy_malloc) || !defined(tommy_calloc) || !defined(tommy_realloc) || !defined(tommy_free) #include @@ -99,7 +154,7 @@ typedef tommy_uint32_t tommy_uint_t; #endif /******************************************************************************/ -/* modificators */ +/* modifiers */ /** \internal * Definition of TOMMY_API. @@ -124,7 +179,7 @@ typedef tommy_uint32_t tommy_uint_t; * Definition of the restrict keyword if available. */ #if !defined(tommy_restrict) -#if __STDC_VERSION__ >= 199901L +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L #define tommy_restrict restrict #elif defined(_MSC_VER) || defined(__GNUC__) #define tommy_restrict __restrict @@ -159,12 +214,12 @@ typedef tommy_uint32_t tommy_uint_t; /* key/hash */ /** - * Type used in indexed data structures to store the key of an object. + * Type used in indexed data structures to store the key of an element. */ typedef tommy_size_t tommy_key_t; /** - * Type used in hashtables to store the hash of an object. + * Type used in hashtables to store the hash of an element. */ typedef tommy_size_t tommy_hash_t; @@ -174,11 +229,11 @@ typedef tommy_size_t tommy_hash_t; /** * Data structure node. * This node type is shared between all the data structures and used to store some - * info directly into the objects you want to store. + * info directly into the elements you want to store. * * A typical declaration is: * \code - * struct object { + * struct element { * tommy_node node; * // other fields * }; @@ -187,7 +242,7 @@ typedef tommy_size_t tommy_hash_t; typedef struct tommy_node_struct { /** * Next node. - * The tail node has it at 0, like a 0 terminated list. + * The tail node has it at 0, like a 0-terminated list. */ struct tommy_node_struct* next; @@ -198,8 +253,9 @@ typedef struct tommy_node_struct { struct tommy_node_struct* prev; /** - * Pointer to the object containing the node. + * Pointer to the element containing the node. * This field is initialized when inserting nodes into a data structure. + * The element pointer passed to insertion functions must not be 0. */ void* data; @@ -207,6 +263,7 @@ typedef struct tommy_node_struct { * Index of the node. * With tries this field is used to store the key. * With hashtables this field is used to store the hash value. + * With trees this field is used to store the parent pointer and balance factor. * With lists this field is not used. */ tommy_size_t index; @@ -217,23 +274,20 @@ typedef struct tommy_node_struct { /** * Compare function for elements. - * \param obj_a Pointer to the first object to compare. - * \param obj_b Pointer to the second object to compare. - * \return <0 if the first element is less than the second, ==0 if equal, >0 if greater. * * This function is like the C strcmp(). * * \code - * struct object { + * struct element { * tommy_node node; * int value; * }; * - * int compare(const void* obj_a, const void* obj_b) + * int compare(const void* elem_a, const void* elem_b) * { - * if (((const struct object*)obj_a)->value < ((const struct object*)obj_b)->value) + * if (((const struct element*)elem_a)->value < ((const struct element*)elem_b)->value) * return -1; - * if (((const struct object*)obj_a)->value > ((const struct object*)obj_b)->value) + * if (((const struct element*)elem_a)->value > ((const struct element*)elem_b)->value) * return 1; * return 0; * } @@ -241,64 +295,62 @@ typedef struct tommy_node_struct { * tommy_list_sort(&list, compare); * \endcode * + * \param elem_a Pointer to the first element to compare. + * \param elem_b Pointer to the second element to compare. + * \return <0 if the first element is less than the second, ==0 if equal, >0 if greater. */ -typedef int tommy_compare_func(const void* obj_a, const void* obj_b); +typedef int tommy_compare_func(const void* elem_a, const void* elem_b); /** * Search function for elements. - * \param arg Pointer to the value to search as passed at the search function. - * \param obj Pointer to the object to compare to. - * \return ==0 if the value matches the element. !=0 if different. - * - * The first argument is a pointer to the value to search exactly - * as it's passed at the search function called. - * The second argument is a pointer to the object inside the hashtable to compare. - * - * The return value has to be 0 if the values are equal. != 0 if they are different. * * \code - * struct object { + * struct element { * tommy_node node; * int value; * }; * - * int compare(const void* arg, const void* obj) + * int compare(const void* arg, const void* elem) * { * const int* value_to_find = arg; - * const struct object* object_to_compare = obj; + * const struct element* element_to_compare = elem; * - * return *value_to_find != object_to_compare->value; + * return *value_to_find != element_to_compare->value; * } * * int value_to_find = 1; - * struct object* obj = tommy_hashtable_search(&hashtable, compare, &value_to_find, tommy_inthash_u32(value_to_find)); - * if (!obj) { + * struct element* elem = tommy_hashtable_search(&hashtable, compare, &value_to_find, tommy_inthash_u32(value_to_find)); + * if (!elem) { * // not found * } else { * // found * } * \endcode * + * \param arg Pointer to the value to search as passed to the search function. + * \param elem Pointer to the element to compare to. + * \return ==0 if the value matches the element. !=0 if different. */ -typedef int tommy_search_func(const void* arg, const void* obj); +typedef int tommy_search_func(const void* arg, const void* elem); /** * Foreach function. - * \param obj Pointer to the object to iterate. * - * A typical example is to use free() to deallocate all the objects in a list. + * A typical example is to use free() to deallocate all the elements in a list. * \code - * tommy_list_foreach(&list, (tommy_foreach_func*)free); + * tommy_list_foreach(&list, free); * \endcode + * + * \param elem Pointer to the element to iterate. */ -typedef void tommy_foreach_func(void* obj); +typedef void tommy_foreach_func(void* elem); /** * Foreach function with an argument. * \param arg Pointer to a generic argument. - * \param obj Pointer to the object to iterate. + * \param elem Pointer to the element to iterate. */ -typedef void tommy_foreach_arg_func(void* arg, void* obj); +typedef void tommy_foreach_arg_func(void* arg, void* elem); /******************************************************************************/ /* bit hacks */ @@ -315,7 +367,7 @@ typedef void tommy_foreach_arg_func(void* arg, void* obj); /** \internal * Integer log2 for constants. - * You can use it only for exact power of 2 up to 256. + * You can use it only for exact powers of 2 up to 256. */ #define TOMMY_ILOG2(value) ((value) == 256 ? 8 : (value) == 128 ? 7 : (value) == 64 ? 6 : (value) == 32 ? 5 : (value) == 16 ? 4 : (value) == 8 ? 3 : (value) == 4 ? 2 : (value) == 2 ? 1 : 0) @@ -355,9 +407,9 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value) */ return __builtin_clz(value) ^ 31; #else - /* Find the log base 2 of an N-bit integer in O(lg(N)) operations with multiply and lookup */ + /* find the log base 2 of an N-bit integer in O(lg(N)) operations with multiply and lookup */ /* from http://graphics.stanford.edu/~seander/bithacks.html */ - static unsigned char TOMMY_DE_BRUIJN_INDEX_ILOG2[32] = { + static const unsigned char TOMMY_DE_BRUIJN_INDEX_ILOG2[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; @@ -372,13 +424,17 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value) #endif } -#if TOMMY_SIZE_BIT == 64 /** * Bit scan reverse or integer log2 for 64 bits. + * Return the bit index of the most significant 1 bit. + * + * If no bit is set, the result is undefined. + * \param value Value to scan. 0 is not allowed. + * \return The index of the most significant bit set. */ tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value) { -#if defined(_MSC_VER) +#if defined(_MSC_VER) && defined(_WIN64) unsigned long count; _BitScanReverse64(&count, value); return count; @@ -393,7 +449,6 @@ tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value) return tommy_ilog2_u32(l); #endif } -#endif /** * Bit scan forward or trailing zero count. @@ -412,24 +467,28 @@ tommy_inline tommy_uint_t tommy_ctz_u32(tommy_uint32_t value) #elif defined(__GNUC__) return __builtin_ctz(value); #else - /* Count the consecutive zero bits (trailing) on the right with multiply and lookup */ + /* count the consecutive zero bits (trailing) on the right with multiply and lookup */ /* from http://graphics.stanford.edu/~seander/bithacks.html */ static const unsigned char TOMMY_DE_BRUIJN_INDEX_CTZ[32] = { 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; - return TOMMY_DE_BRUIJN_INDEX_CTZ[(tommy_uint32_t)(((value & - value) * 0x077CB531U)) >> 27]; + return TOMMY_DE_BRUIJN_INDEX_CTZ[(tommy_uint32_t)(((value & -value) * 0x077CB531U)) >> 27]; #endif } -#if TOMMY_SIZE_BIT == 64 /** * Bit scan forward or trailing zero count for 64 bits. + * Return the bit index of the least significant 1 bit. + * + * If no bit is set, the result is undefined. + * \param value Value to scan. 0 is not allowed. + * \return The index of the least significant bit set. */ tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value) { -#if defined(_MSC_VER) +#if defined(_MSC_VER) && defined(_WIN64) unsigned long count; _BitScanForward64(&count, value); return count; @@ -444,16 +503,16 @@ tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value) return tommy_ctz_u32(h) + 32; #endif } -#endif /** * Rounds up to the next power of 2. * For the value 0, the result is undefined. + * \param value Value to round up. * \return The smallest power of 2 not less than the specified value. */ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) { - /* Round up to the next highest power of 2 */ + /* round up to the next highest power of 2 */ /* from http://graphics.stanford.edu/~seander/bithacks.html */ --value; @@ -469,6 +528,9 @@ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) /** * Rounds up to the next power of 2 for 64 bits. + * For the value 0, the result is undefined. + * \param value Value to round up. + * \return The smallest power of 2 not less than the specified value. */ tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value) { @@ -485,7 +547,8 @@ tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value) } /** - * Check if the specified word has a byte at 0. + * Checks if the specified word has a byte at 0. + * \param value Value to check. * \return 0 or 1. */ tommy_inline int tommy_haszero_u32(tommy_uint32_t value) @@ -493,7 +556,17 @@ tommy_inline int tommy_haszero_u32(tommy_uint32_t value) return ((value - 0x01010101) & ~value & 0x80808080) != 0; } -/* +/** + * Checks if the specified 64-bit word has a byte at 0. + * \param value Value to check. + * \return 0 or 1. + */ +tommy_inline int tommy_haszero_u64(tommy_uint64_t value) +{ + return ((value - 0x0101010101010101ULL) & ~value & 0x8080808080808080ULL) != 0; +} + +/** * Bit depth mapping. */ #if TOMMY_SIZE_BIT == 64 @@ -507,3 +580,4 @@ tommy_inline int tommy_haszero_u32(tommy_uint32_t value) #endif #endif + From 3b8c8c10e7566673c5697d667cd376d9cb9ba745 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 7 Oct 2026 19:13:27 +0200 Subject: [PATCH 31/51] Use 64-bit TommyDS hash functions on 64-bit architectures --- cmdline/dup.c | 14 +++++++++----- cmdline/elem.h | 34 +++++++++++++++++++++++++--------- cmdline/pool.c | 6 +++++- cmdline/scan.c | 2 +- cmdline/search.c | 4 ++-- 5 files changed, 42 insertions(+), 18 deletions(-) diff --git a/cmdline/dup.c b/cmdline/dup.c index 64e6352c..c1653e40 100644 --- a/cmdline/dup.c +++ b/cmdline/dup.c @@ -70,9 +70,13 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d return hash; } -static inline tommy_uint32_t hash_hash(struct snapraid_hash* hash) +static inline tommy_hash_t hash_hash(struct snapraid_hash* hash) { +#if TOMMY_SIZE_BIT == 64 + return tommy_hash_u64(0, hash->hash, HASH_MAX); +#else return tommy_hash_u32(0, hash->hash, HASH_MAX); +#endif } static void hash_free(void* void_hash) @@ -112,7 +116,7 @@ void state_dup(struct snapraid_state* state) for (j = disk->filelist; j != 0; j = j->next) { struct snapraid_file* file = j->data; struct snapraid_hash* hash; - tommy_hash_t hash32; + tommy_hash_t hash_val; /* if empty, skip it */ if (file->size == 0) @@ -124,9 +128,9 @@ void state_dup(struct snapraid_state* state) if (!hash) continue; - hash32 = hash_hash(hash); + hash_val = hash_hash(hash); - struct snapraid_hash* found = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash32); + struct snapraid_hash* found = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash_val); if (found) { ++count; size += found->file->size; @@ -134,7 +138,7 @@ void state_dup(struct snapraid_state* state) printf("%12" PRIu64 " %s = %s\n", file->size, fmt_term(disk, file->sub), fmt_term(found->disk, found->file->sub)); hash_free(hash); } else { - tommy_hashdyn_insert(&hashset, &hash->node, hash, hash32); + tommy_hashdyn_insert(&hashset, &hash->node, hash, hash_val); } } } diff --git a/cmdline/elem.h b/cmdline/elem.h index 118544ef..fc253c14 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -982,9 +982,9 @@ int file_path_compare(const void* void_a, const void* void_b); /** * Compute the hash of a file inode. */ -static inline tommy_uint32_t file_inode_hash(uint64_t inode) +static inline tommy_hash_t file_inode_hash(uint64_t inode) { - return (tommy_uint32_t)tommy_inthash_u64(inode); + return tommy_inthash_u64(inode); } /** @@ -1003,15 +1003,19 @@ int file_pathstamp_compare(const void* void_a, const void* void_b); /** * Compute the hash of a file path with known length. */ -static inline tommy_uint32_t file_path_hash_len(const char* sub, size_t len) +static inline tommy_hash_t file_path_hash_len(const char* sub, size_t len) { +#if TOMMY_SIZE_BIT == 64 + return tommy_hash_u64(0, sub, len); +#else return tommy_hash_u32(0, sub, len); +#endif } /** * Compute the hash of a file path. */ -static inline tommy_uint32_t file_path_hash(const char* sub) +static inline tommy_hash_t file_path_hash(const char* sub) { return file_path_hash_len(sub, strlen(sub)); } @@ -1019,9 +1023,13 @@ static inline tommy_uint32_t file_path_hash(const char* sub) /** * Compute the hash of a file stamp. */ -static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec) +static inline tommy_hash_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec) { +#if TOMMY_SIZE_BIT == 64 + return tommy_inthash_u64((uint64_t)size ^ tommy_inthash_u64((uint64_t)mtime_sec ^ (uint64_t)(uint32_t)mtime_nsec)); +#else return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32((tommy_uint32_t)mtime_sec ^ tommy_inthash_u32(mtime_nsec))); +#endif } /** @@ -1097,15 +1105,19 @@ int link_alpha_compare(const void* void_a, const void* void_b); /** * Compute the hash of a link name with known length. */ -static inline tommy_uint32_t link_name_hash_len(const char* name, size_t len) +static inline tommy_hash_t link_name_hash_len(const char* name, size_t len) { +#if TOMMY_SIZE_BIT == 64 + return tommy_hash_u64(0, name, len); +#else return tommy_hash_u32(0, name, len); +#endif } /** * Compute the hash of a link name. */ -static inline tommy_uint32_t link_name_hash(const char* name) +static inline tommy_hash_t link_name_hash(const char* name) { return link_name_hash_len(name, strlen(name)); } @@ -1147,15 +1159,19 @@ int dir_name_compare(const void* void_arg, const void* void_data); /** * Compute the hash of a dir name with known length. */ -static inline tommy_uint32_t dir_name_hash_len(const char* name, size_t len) +static inline tommy_hash_t dir_name_hash_len(const char* name, size_t len) { +#if TOMMY_SIZE_BIT == 64 + return tommy_hash_u64(0, name, len); +#else return tommy_hash_u32(0, name, len); +#endif } /** * Compute the hash of a dir name. */ -static inline tommy_uint32_t dir_name_hash(const char* name) +static inline tommy_hash_t dir_name_hash(const char* name) { return dir_name_hash_len(name, strlen(name)); } diff --git a/cmdline/pool.c b/cmdline/pool.c index 5a46fd49..a386f623 100644 --- a/cmdline/pool.c +++ b/cmdline/pool.c @@ -30,9 +30,13 @@ struct snapraid_pool* pool_alloc(const char* dir, const char* name, const char* return pool; } -static inline tommy_uint32_t pool_hash(const char* file) +static inline tommy_hash_t pool_hash(const char* file) { +#if TOMMY_SIZE_BIT == 64 + return tommy_hash_u64(0, file, strlen(file)); +#else return tommy_hash_u32(0, file, strlen(file)); +#endif } static void pool_free(void* void_pool) diff --git a/cmdline/scan.c b/cmdline/scan.c index a2376b56..cc316bc4 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -1021,7 +1021,7 @@ static void scan_file_apply(void* void_scan, void* void_disc) * in such case we have a "copy" over an "update" */ if (!state->opt.force_nocopy) { - tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec); + tommy_hash_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec); /* search for a file with the same name and stamp in all the disks */ for (i = state->disklist; i != 0; i = i->next) { diff --git a/cmdline/search.c b/cmdline/search.c index bc4fdcdd..943802e8 100644 --- a/cmdline/search.c +++ b/cmdline/search.c @@ -22,7 +22,7 @@ static const char* es(int err) static void search_file(struct snapraid_state* state, const char* path, data_off_t size, int64_t mtime_sec, int mtime_nsec) { struct snapraid_search_file* file; - tommy_uint32_t file_hash; + tommy_hash_t file_hash; file = malloc_nofail(sizeof(struct snapraid_search_file)); file->path = strdup_nofail(path); @@ -133,7 +133,7 @@ int search_file_compare(const void* void_arg, const void* void_data) int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer) { struct snapraid_search_file* file; - tommy_uint32_t file_hash; + tommy_hash_t file_hash; struct search_file_compare_arg arg; arg.state = state; From f775075bc524ec592908b80c17028aac4cf751c3 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 11 Oct 2026 23:28:55 +0200 Subject: [PATCH 32/51] Synchronize file metadata for new and modified files during scan on Windows --- cmdline/scan.c | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/cmdline/scan.c b/cmdline/scan.c index cc316bc4..1664ca05 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -20,8 +20,8 @@ static const char* es(int err) * If the file metadata (size and timestamp) is known to be synchronized with disk. * * Always set on POSIX where stat()/lstat() returns coherent metadata. On Windows NTFS, - * directory enumeration may return stale size/mtime for hardlinks until synchronized - * via lstat_sync() (GetFileInformationByHandle). + * directory enumeration may return stale size/mtime for open/modified files and hardlinks + * until synchronized via lstat_sync() (GetFileInformationByHandle). */ #define DISCOVERY_IS_SCAN_SYNCED 0x1 @@ -1655,13 +1655,24 @@ static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* pa if (!st) st = DSTAT(path_next, dd, &st_buf); + existing = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub_next, file_path_hash(sub_next)); + if (existing) + file_flag_set(existing, FILE_IS_DISCOVERED); + #if HAVE_LSTAT_SYNC /* * In Windows fast directory enumeration, st_ino may be missing (0). - * Only st_ino is required for file identity and hardlink tracking; - * we call lstat_sync() only when st_ino is missing to avoid expensive file opens. + * In addition, directory enumeration may report stale size and mtime + * for files currently open or being written to. + * Refresh via lstat_sync() if st_ino is missing, or if the metadata + * is not synchronized and the file is new or modified. */ - if (st->st_ino == INODE_INVALID) { + if (st->st_ino == INODE_INVALID + || (st->st_sync == 0 + && (!existing + || existing->size != st->st_size + || existing->mtime_sec != st->st_mtime + || existing->mtime_nsec != STAT_NSEC(st)))) { if (lstat_sync(path_next, st, 0) != 0) { /* LCOV_EXCL_START */ log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno)); @@ -1672,10 +1683,6 @@ static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* pa } #endif - existing = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub_next, file_path_hash(sub_next)); - if (existing) - file_flag_set(existing, FILE_IS_DISCOVERED); - scan_file_discover(scan, sub_next, st); processed = 1; } else { @@ -1937,8 +1944,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) * - Each disk is read only by its scan thread, enumerating files and populating * file_discovery_list without observable SnapRAID classification side effects. * - No old file, link, directory, or parity allocation is removed. - * - Inode collisions on volatile-hardlink filesystems trigger selective lstat_sync() - * to normalize authoritative metadata across hardlink groups before Apply. + * - New/modified files and inode collisions on volatile filesystems trigger selective + * lstat_sync() to obtain authoritative metadata before Apply. */ for (i = scanlist; i != 0; i = i->next) { struct snapraid_scan* scan = i->data; From cc18ed39c6e9a9fb6601f109e862ddb513268c5a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 12 Oct 2026 10:57:51 +0200 Subject: [PATCH 33/51] Keep generic license texts without embedded copyright notices --- LICENSES/BSD-2-Clause | 2 -- 1 file changed, 2 deletions(-) diff --git a/LICENSES/BSD-2-Clause b/LICENSES/BSD-2-Clause index 6c86157a..6535ce92 100644 --- a/LICENSES/BSD-2-Clause +++ b/LICENSES/BSD-2-Clause @@ -1,5 +1,3 @@ -Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. - Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From 25c86bc1e7b02c2b58987e6f2bdbc25a9efb3e41 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 13 Oct 2026 11:20:19 +0200 Subject: [PATCH 34/51] Remove leading double underscores from header include guards --- AGENTS.md | 2 +- cmdline/app.h | 4 ++-- cmdline/bw.h | 4 ++-- cmdline/elem.h | 4 ++-- cmdline/handle.h | 4 ++-- cmdline/import.h | 4 ++-- cmdline/io.h | 4 ++-- cmdline/locate.h | 4 ++-- cmdline/memory.h | 4 ++-- cmdline/parity.h | 4 ++-- cmdline/search.h | 4 ++-- cmdline/snapraid.h | 4 ++-- cmdline/state.h | 4 ++-- cmdline/str.h | 4 ++-- cmdline/stream.h | 4 ++-- cmdline/support.h | 4 ++-- cmdline/thermal.h | 4 ++-- cmdline/util.h | 4 ++-- raid/combo.h | 4 ++-- raid/cpu.h | 4 ++-- raid/gf.h | 4 ++-- raid/helper.h | 4 ++-- raid/internal.h | 4 ++-- raid/memory.h | 4 ++-- raid/raid.h | 4 ++-- raid/test.h | 4 ++-- 26 files changed, 51 insertions(+), 51 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b0d4244e..bc96d77a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,7 +107,7 @@ SnapRAID is designed to recover from an interruption, including `SIGKILL`, at an - **Indentation**: Tabs for indentation, no alignment (existing codebase style) - **Comments**: Use `/** ... */` for multiline top comments preceding declarations and `/* ... */` for multiline comments within code. Use `/* first letter lowercase */` for single-line inline notes. - **Critical Comments**: Always add a comment at non-obvious critical points, especially around data-integrity invariants, crash recovery, fallback behavior, concurrency, and fatal versus best-effort error handling. Explain why the logic is required, not merely what the code does. -- **Headers**: All `.h` files have include guards (`#ifndef __NAME_H`) +- **Headers**: All `.h` files have include guards (`#ifndef NAME_H`) - **Preferences**: Use 0 instead of NULL and '\0' - **Preferences**: Use prefix ++variable and --variable instead of postfix variable++ and variable-- where both are equivalent - **Safety Checks**: Avoid adding safety checks for conditions that never happen diff --git a/cmdline/app.h b/cmdline/app.h index 4c3ceda2..69a2cdf4 100644 --- a/cmdline/app.h +++ b/cmdline/app.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2026 Andrea Mazzoleni -#ifndef __APP_H -#define __APP_H +#ifndef APP_H +#define APP_H #include "os/os.h" diff --git a/cmdline/bw.h b/cmdline/bw.h index 9ba69e12..482202a5 100644 --- a/cmdline/bw.h +++ b/cmdline/bw.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2016 Andrea Mazzoleni -#ifndef __BW_H -#define __BW_H +#ifndef BW_H +#define BW_H #include "state.h" #include "support.h" diff --git a/cmdline/elem.h b/cmdline/elem.h index fc253c14..85edbf05 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __ELEM_H -#define __ELEM_H +#ifndef ELEM_H +#define ELEM_H #include "util.h" #include "support.h" diff --git a/cmdline/handle.h b/cmdline/handle.h index f117f6b8..757c939d 100644 --- a/cmdline/handle.h +++ b/cmdline/handle.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __HANDLE_H -#define __HANDLE_H +#ifndef HANDLE_H +#define HANDLE_H #include "state.h" #include "support.h" diff --git a/cmdline/import.h b/cmdline/import.h index 8aef9b0d..4676063f 100644 --- a/cmdline/import.h +++ b/cmdline/import.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __IMPORT_H -#define __IMPORT_H +#ifndef IMPORT_H +#define IMPORT_H #include "elem.h" #include "state.h" diff --git a/cmdline/io.h b/cmdline/io.h index df535946..92f4861b 100644 --- a/cmdline/io.h +++ b/cmdline/io.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2016 Andrea Mazzoleni -#ifndef __IO_H -#define __IO_H +#ifndef IO_H +#define IO_H #include "state.h" #include "support.h" diff --git a/cmdline/locate.h b/cmdline/locate.h index 57162559..dc3a6cf9 100644 --- a/cmdline/locate.h +++ b/cmdline/locate.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2026 Andrea Mazzoleni -#ifndef __LOCATE_H -#define __LOCATE_H +#ifndef LOCATE_H +#define LOCATE_H /****************************************************************************/ /* snapraid */ diff --git a/cmdline/memory.h b/cmdline/memory.h index ef3e187a..88e6e278 100644 --- a/cmdline/memory.h +++ b/cmdline/memory.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __MEMORY_H -#define __MEMORY_H +#ifndef MEMORY_H +#define MEMORY_H /****************************************************************************/ /* memory */ diff --git a/cmdline/parity.h b/cmdline/parity.h index dab95dd7..91140492 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __PARITY_H -#define __PARITY_H +#ifndef PARITY_H +#define PARITY_H #include "support.h" #include "bw.h" diff --git a/cmdline/search.h b/cmdline/search.h index 55c8fef3..dc84747e 100644 --- a/cmdline/search.h +++ b/cmdline/search.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2014 Andrea Mazzoleni -#ifndef __SEARCH_H -#define __SEARCH_H +#ifndef SEARCH_H +#define SEARCH_H #include "elem.h" #include "state.h" diff --git a/cmdline/snapraid.h b/cmdline/snapraid.h index 3bff83ec..2b2c9eb4 100644 --- a/cmdline/snapraid.h +++ b/cmdline/snapraid.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __SNAPRAID_H -#define __SNAPRAID_H +#ifndef SNAPRAID_H +#define SNAPRAID_H /****************************************************************************/ /* snapraid */ diff --git a/cmdline/state.h b/cmdline/state.h index f9eb37ce..31215e86 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __STATE_H -#define __STATE_H +#ifndef STATE_H +#define STATE_H #include "elem.h" diff --git a/cmdline/str.h b/cmdline/str.h index bbc4f072..976ae435 100644 --- a/cmdline/str.h +++ b/cmdline/str.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2025 Andrea Mazzoleni -#ifndef __STR_H -#define __STR_H +#ifndef STR_H +#define STR_H #include "app.h" #include "str.h" diff --git a/cmdline/stream.h b/cmdline/stream.h index 08377adc..b39b98c0 100644 --- a/cmdline/stream.h +++ b/cmdline/stream.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __STREAM_H -#define __STREAM_H +#ifndef STREAM_H +#define STREAM_H #include "support.h" #include "util.h" diff --git a/cmdline/support.h b/cmdline/support.h index 34f945b0..ec05ed9e 100644 --- a/cmdline/support.h +++ b/cmdline/support.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __SUPPORT_H -#define __SUPPORT_H +#ifndef SUPPORT_H +#define SUPPORT_H #include "app.h" #include "memory.h" diff --git a/cmdline/thermal.h b/cmdline/thermal.h index 5e23ddb3..5ae21587 100644 --- a/cmdline/thermal.h +++ b/cmdline/thermal.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2025 Andrea Mazzoleni -#ifndef __THERMAL_H -#define __THERMAL_H +#ifndef THERMAL_H +#define THERMAL_H #include "elem.h" diff --git a/cmdline/util.h b/cmdline/util.h index c3f282a7..b9b4f25e 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef __UTIL_H -#define __UTIL_H +#ifndef UTIL_H +#define UTIL_H #include "tommyds/tommytypes.h" diff --git a/raid/combo.h b/raid/combo.h index ff59be03..7c833430 100644 --- a/raid/combo.h +++ b/raid/combo.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_COMBO_H -#define __RAID_COMBO_H +#ifndef RAID_COMBO_H +#define RAID_COMBO_H #include diff --git a/raid/cpu.h b/raid/cpu.h index 0933133a..418c1c05 100644 --- a/raid/cpu.h +++ b/raid/cpu.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_CPU_H -#define __RAID_CPU_H +#ifndef RAID_CPU_H +#define RAID_CPU_H #ifdef CONFIG_X86 diff --git a/raid/gf.h b/raid/gf.h index 0e92d508..dce476cb 100644 --- a/raid/gf.h +++ b/raid/gf.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_GF_H -#define __RAID_GF_H +#ifndef RAID_GF_H +#define RAID_GF_H /* * Galois field operations. diff --git a/raid/helper.h b/raid/helper.h index f9ca609e..15cbaa46 100644 --- a/raid/helper.h +++ b/raid/helper.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_HELPER_H -#define __RAID_HELPER_H +#ifndef RAID_HELPER_H +#define RAID_HELPER_H /** * Inserts an integer in a sorted vector. diff --git a/raid/internal.h b/raid/internal.h index 0314c95a..3650665f 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_INTERNAL_H -#define __RAID_INTERNAL_H +#ifndef RAID_INTERNAL_H +#define RAID_INTERNAL_H #if HAVE_CONFIG_H /* Includes the project configuration for HAVE_* defines */ diff --git a/raid/memory.h b/raid/memory.h index 19aa8af1..7d73e0b3 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_MEMORY_H -#define __RAID_MEMORY_H +#ifndef RAID_MEMORY_H +#define RAID_MEMORY_H /** * Memory allocation and benchmark layout helpers. diff --git a/raid/raid.h b/raid/raid.h index d4595a87..544d7cda 100644 --- a/raid/raid.h +++ b/raid/raid.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_H -#define __RAID_H +#ifndef RAID_H +#define RAID_H #include diff --git a/raid/test.h b/raid/test.h index bda0b456..1c0c99b4 100644 --- a/raid/test.h +++ b/raid/test.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef __RAID_TEST_H -#define __RAID_TEST_H +#ifndef RAID_TEST_H +#define RAID_TEST_H /** * Tests insertion function. From 4a9871d08789e277386704c272d6494bd070af96 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 14 Oct 2026 18:46:56 +0200 Subject: [PATCH 35/51] Use mulhash for integer keys while retaining inthash_u64 on 32-bit architectures --- cmdline/elem.c | 2 +- cmdline/elem.h | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index db02a55c..c349a002 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -1773,7 +1773,7 @@ void bucket_free(void* void_bucket) void bucket_insert(tommy_hashdyn* bucket_hash, time_t time_at, block_off_t count, int justsynced) { - tommy_hash_t hash = tommy_inthash_u64(time_at); + tommy_hash_t hash = bucket_time_hash(time_at); tommy_node* i = tommy_hashdyn_bucket(bucket_hash, hash); while (i) { struct snapraid_bucket* entry = i->data; diff --git a/cmdline/elem.h b/cmdline/elem.h index 85edbf05..7e1ef4a0 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -984,7 +984,12 @@ int file_path_compare(const void* void_a, const void* void_b); */ static inline tommy_hash_t file_inode_hash(uint64_t inode) { +#if TOMMY_SIZE_BIT == 64 + return tommy_mulhash_u64(inode); +#else + /* avoid expensive 64-bit multiplications on 32-bit architectures */ return tommy_inthash_u64(inode); +#endif } /** @@ -1025,10 +1030,20 @@ static inline tommy_hash_t file_path_hash(const char* sub) */ static inline tommy_hash_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec) { + tommy_uint32_t nsec = mtime_nsec; + #if TOMMY_SIZE_BIT == 64 - return tommy_inthash_u64((uint64_t)size ^ tommy_inthash_u64((uint64_t)mtime_sec ^ (uint64_t)(uint32_t)mtime_nsec)); + tommy_uint64_t size_key = size; + tommy_uint64_t sec_key = mtime_sec; + tommy_uint64_t nsec_key = nsec; + + /* valid nanoseconds use 30 bits above contemporary seconds; -1 stays distinct */ + return tommy_mulhash_u64(size_key ^ tommy_mulhash_u64(sec_key ^ (nsec_key << 34))); #else - return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32((tommy_uint32_t)mtime_sec ^ tommy_inthash_u32(mtime_nsec))); + tommy_uint32_t size_key = size; + tommy_uint32_t sec_key = mtime_sec; + + return tommy_mulhash_u32(size_key ^ tommy_mulhash_u32(sec_key) ^ tommy_mulhash_u32(nsec)); #endif } @@ -1556,6 +1571,19 @@ struct snapraid_bucket { void bucket_free(void* void_bucket); +/** + * Compute the hash of a bucket time. + */ +static inline tommy_hash_t bucket_time_hash(time_t time_at) +{ +#if TOMMY_SIZE_BIT == 64 + return tommy_mulhash_u64(time_at); +#else + /* avoid expensive 64-bit multiplications on 32-bit architectures */ + return tommy_inthash_u64(time_at); +#endif +} + /** * Insert an bucket record into the hashtable */ From 20e081d5ac75a4082943cafefce51a297f7fd4a6 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 17 Oct 2026 21:23:59 +0200 Subject: [PATCH 36/51] Update CHECKSUMS --- CHECKSUMS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHECKSUMS b/CHECKSUMS index fad5b785..8d59eba8 100644 --- a/CHECKSUMS +++ b/CHECKSUMS @@ -221,12 +221,18 @@ e11c452e2adec376e6fcaee9ff2ec4de4e943c35690229c9c67056c5eaed52a1 snapraid-14.9- 6e8892175bfe240ae7828d513342180a249511a2d5baf8c003ae8b2bb82e0f46 snapraid-14.10-windows-x64.zip 1ec0572359fb31f2827702d792db0bdf6a95161eb9e2a6deff2bdc7e1f5b794b snapraid-14.10-windows-x86.zip 6930d629c73b5307ec32277310ed13887b1bd051e30dc41c75b4752040032d1c snapraid-14.10-x86_64-1_am.txz +ceba44e15566386d923d6895ecab81f7437848b03a4cf9585c02d3636197ce19 snapraid-15.0.tar.gz 1e15cc11b26b02fbb54a679c0379161bf3c94aadfaf29081d876990d809a43f9 snapraid-15.0rc2.tar.gz a546944800a5001c3960715b97ba73cae219c854da1bb55dca16ce70fe06c6d6 snapraid-15.0rc2-1-x86_64.pkg.tar.zst 1327c122a66f91e7780e77431a318907661e690a4aca7fa78f9d504e8dde9e06 snapraid-15.0rc2-1.x86_64.rpm 092d1e91915e7644288805d1a91bd6f20ee0591d70157f5bb5980e26eaf82c1a snapraid-15.0rc2-windows-x64.zip 2c010c4ea69f39d28f266d978d78e0b2c3a845ab47f847cb5a305fc242994691 snapraid-15.0rc2-windows-x86.zip 7ada0dd95bad90895c3faaf627f99da02a4e31ac948d05f012efd2ab05a37d97 snapraid-15.0rc2-x86_64-1_am.txz +9d55d0308ce41beb29956b59f9d2e32d419c930118266d3e08b15f658119f4fb snapraid-15.0-1-x86_64.pkg.tar.zst +ad66c0b4c016775c9a759adf2a5202bebb357d319b0063b9cf68befde9a67363 snapraid-15.0-1.x86_64.rpm +704706c21b432719c4da8cd237c1f26327af89c86604dc339a38d63574004a22 snapraid-15.0-windows-x64.zip +58b3ed90307e41b7a6e862d2dc070204c98b5c3ac24718e2fb9fe03e275bc2ce snapraid-15.0-windows-x86.zip +9483df5a39a8b59c32433331ac92a4eef54dcff4722694a5bc5e89bcefb6b3b0 snapraid-15.0-x86_64-1_am.txz 128b5b0f80a6a0911e77ed3b5d9d81529f5ea1d8fffbb5112e3af3467d80464a snapraid_14.0-1_amd64.deb b9c41ae2e8edbc413abd3172b7ed2da2cba7fa0e631d0f47fc1dafccf7c6e92c snapraid_14.1-1_amd64.deb e6456b926bb053d34645e31e5e3d8363e952c9869e873eb348abb1149598d331 snapraid_14.2-1_amd64.deb @@ -239,3 +245,4 @@ c0e00a25b9bce40ff74f5a55401c3bfedac1881fd99aaa90a692f5858d417f7d snapraid_14.8- 09064689a64d965ac33ee89fb8b2cc14dfec0745f2b3b86a78f377fc353b7e07 snapraid_14.9-1_amd64.deb 15f5fd9f639912c94daae9d75d3567b7a3fc0e18475880f972acd8d8e9e0992c snapraid_14.10-1_amd64.deb fc875789267ec9b057d14780f04362cc32b3cb4191e707d6e0390f05ff346d46 snapraid_15.0rc2-1_amd64.deb +845091e7eeb2495fc3899dc6e8fab2c3fe7a5945f345c767a7603dab90e2cb84 snapraid_15.0-1_amd64.deb From a0d6bc288a8c3ba966bd54076b23f06ef5e48a1f Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 18 Oct 2026 10:45:14 +0200 Subject: [PATCH 37/51] Fix touch abort before the first sync when no content file exists --- cmdline/snapraid.c | 16 +++++++++++++++- cmdline/state.c | 31 +++++++++++++++++++++++++++++++ cmdline/state.h | 10 ++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index c3c2f258..598a90fe 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -1530,7 +1530,8 @@ int snapraid_main(int argc, char* argv[]) if (state.opt.force_realloc) state_locate_mark_tail_blocks_for_resync(&state, opt.parity_tail); - if (opt.gui_touch_before) { + /* skip touch itself until content is loaded and sync has initialized parity sizes */ + if (opt.gui_touch_before && state_is_write_ready(&state)) { ret = state_touch(&state); /* save the new state if required */ @@ -1627,6 +1628,9 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_REHASH) { state_read(&state); + /* prevent saving content with uninitialized parity split sizes */ + state_require_write_ready(&state); + state_rehash(&state); /* @@ -1645,6 +1649,9 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_SCRUB) { state_read(&state); + /* prevent saving content with uninitialized parity split sizes */ + state_require_write_ready(&state); + /* refresh the size info before the content write */ state_refresh(&state); @@ -1667,6 +1674,9 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_REWRITE) { state_read(&state); + /* prevent saving content with uninitialized parity split sizes */ + state_require_write_ready(&state); + /* intercept signals while operating */ os_signal_init(app_signal_handler, app_signal_handler); @@ -1682,6 +1692,9 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_TOUCH) { state_read(&state); + /* prevent saving content with uninitialized parity split sizes */ + state_require_write_ready(&state); + memory(); ret = state_touch(&state); @@ -1696,6 +1709,7 @@ int snapraid_main(int argc, char* argv[]) */ os_signal_init(app_signal_handler, app_signal_handler); + /* save the new state if required */ if (state.need_write) state_write(&state); } else if (operation == OPERATION_SPINUP) { diff --git a/cmdline/state.c b/cmdline/state.c index c7be0926..0d44fa28 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -5428,6 +5428,37 @@ void state_read(struct snapraid_state* state) state->checked_read = 1; } +/** + * Check if all configured parity splits have persistent sizes. + */ +int state_is_write_ready(struct snapraid_state* state) +{ + unsigned l, s; + + for (l = 0; l < state->level; ++l) { + for (s = 0; s < state->parity[l].split_mac; ++s) { + if (state->parity[l].split_map[s].size == PARITY_SIZE_INVALID) + return 0; + } + } + + return 1; +} + +/** + * Require sync to initialize parity sizes before commands that can save content. + * Check before modifying timestamps or metadata, as a later write could fail + * even if the command has no blocks to process (e.g. a missing content replica). + */ +void state_require_write_ready(struct snapraid_state* state) +{ + if (!state_is_write_ready(state)) { + log_fatal(EUSER, "The array has not been synced yet, or its parity configuration has changed.\n"); + log_fatal(EUSER, "Run 'snapraid sync' before '%s'.\n", state->command); + exit(EXIT_FAILURE); + } +} + struct state_verify_thread_context { struct snapraid_state* state; struct snapraid_content* content; diff --git a/cmdline/state.h b/cmdline/state.h index 31215e86..b914b782 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -240,6 +240,16 @@ void state_probe(struct snapraid_state* state); */ void state_read(struct snapraid_state* state); +/** + * Check if all configured parity splits have persistent sizes. + */ +int state_is_write_ready(struct snapraid_state* state); + +/** + * Require initialized parity split sizes before commands that can save content. + */ +void state_require_write_ready(struct snapraid_state* state); + /** * Count the number of unsynced parity positions using an event sweep across extents. * Counts positions in [blockmin, blockmax). The caller must ensure that the disk list, From 8a5b2adb75599d7afd3754c91874f217f26db7a4 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 18 Oct 2026 11:23:04 +0200 Subject: [PATCH 38/51] Remove unused checked_read state flag --- cmdline/state.c | 5 ----- cmdline/state.h | 1 - 2 files changed, 6 deletions(-) diff --git a/cmdline/state.c b/cmdline/state.c index 0d44fa28..006a912d 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -171,7 +171,6 @@ void state_init(struct snapraid_state* state) state->autosave = 0; state->need_write = 0; state->written = 0; - state->checked_read = 0; state->block_size = 256 * KIBI; /* default 256 KiB */ state->raid_mode = RAID_MODE_CAUCHY_RAID; state->file_mode = ADVISE_DEFAULT; @@ -5423,9 +5422,6 @@ void state_read(struct snapraid_state* state) log_tag("content_info:read_unixtime:%" PRId64 "\n", (int64_t)st.st_mtime); } log_flush(); - - /* mark that we read the content file, and it passed all the checks */ - state->checked_read = 1; } /** @@ -5967,7 +5963,6 @@ void state_write(struct snapraid_state* state) log_flush(); state->need_write = 0; /* no write needed anymore */ - state->checked_read = 0; /* what we wrote is not checked in read */ state->written = 1; } diff --git a/cmdline/state.h b/cmdline/state.h index b914b782..d9a93f29 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -130,7 +130,6 @@ struct snapraid_state { uint64_t autosave; /**< Autosave after the specified amount of data. 0 to disable. */ int need_write; /**< If the state is changed. */ int written; /**< If the state was written at least one time */ - int checked_read; /**< If the state was read and checked. */ uint32_t block_size; /**< Block size in bytes. */ unsigned raid_mode; /**< Raid mode to use. RAID_MODE_DEFAULT or RAID_MODE_ALTERNATE. */ int file_mode; /**< File access mode. Combination of ADVISE_* flags. */ From 7878bc3d3dd11f9e78a4215473da527db3f137d5 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 18 Oct 2026 22:19:56 +0200 Subject: [PATCH 39/51] Update TODO --- TODO | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/TODO b/TODO index 4623ed47..614d56ef 100644 --- a/TODO +++ b/TODO @@ -6,19 +6,23 @@ This is the list of TODO items for SnapRAID. 16.x Releases ============= -* Use statx() instead of lstat() when scanning directories. +* Support a `[power_control: off]` tag in `smartctl` options to disable power + management (`up` and `down`) on specific disks (e.g. for SAS drives that + fail to wake up on write). Emit an attribute tag in probe for snapraid-daemon. + +* Enable MuseAir and disable random seed * Add a new 'restore' command, similar to 'fix', that recovers a whole disk using the same multithreaded approach as sync/scrub. +* Use statx() instead of lstat() when scanning directories. + * Integration with MergerFS ? * Restore ownership and permissions, at least in Unix. * Restore directory timestamps. -* Enable MuseAir and disable random seed - * Check Reliability Test. We can derive Failure probability from MBTF https://github.com/linuxhw/SMART https://github.com/linuxhw/EnterpriseDrive From 819bb75eaad447f825b452e087a0de7c571fbb85 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 18 Oct 2026 22:30:58 +0200 Subject: [PATCH 40/51] Make test and doc variables in Makefile.am compatible with BSD make --- .github/workflows/build.yml | 8 ++++---- Makefile.am | 32 ++++++++++++++++++++++++++------ 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a8c2d41a..ed5b2e2d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -90,12 +90,12 @@ jobs: with: usesh: true prepare: | - pkg install -y autoconf automake gmake + pkg install -y autoconf automake run: | autoreconf -i - ./configure --enable-warning-as-error MAKE=gmake - gmake all -j$(sysctl -n hw.ncpu) - gmake dist check + ./configure --enable-warning-as-error + make all -j$(sysctl -n hw.ncpu) + make dist check build-musl: runs-on: ubuntu-latest container: alpine:latest diff --git a/Makefile.am b/Makefile.am index 5d2a06a2..1da5e4b6 100644 --- a/Makefile.am +++ b/Makefile.am @@ -228,13 +228,28 @@ dist-hook: LANG_CODES = de es fr it ja ko pl pt ro ru sv uk zh -LIST_D = doc/snapraid.d doc/snapraid_log.d $(patsubst %, doc/snapraid-%.d, $(LANG_CODES)) +LIST_LANG_D = \ + doc/snapraid-de.d \ + doc/snapraid-es.d \ + doc/snapraid-fr.d \ + doc/snapraid-it.d \ + doc/snapraid-ja.d \ + doc/snapraid-ko.d \ + doc/snapraid-pl.d \ + doc/snapraid-pt.d \ + doc/snapraid-ro.d \ + doc/snapraid-ru.d \ + doc/snapraid-sv.d \ + doc/snapraid-uk.d \ + doc/snapraid-zh.d -LIST_MAN = doc/snapraid.1 doc/snapraid_log.1 $(patsubst %, doc/snapraid-%.1, $(LANG_CODES)) +LIST_D = doc/snapraid.d doc/snapraid_log.d $(LIST_LANG_D) -LIST_TXT = doc/snapraid.txt doc/snapraid_log.txt $(patsubst %, doc/snapraid-%.txt, $(LANG_CODES)) +LIST_MAN = $(LIST_D:.d=.1) -LIST_XML = doc/snapraid.xml doc/snapraid_log.xml $(patsubst %, doc/snapraid-%.xml, $(LANG_CODES)) +LIST_TXT = $(LIST_D:.d=.txt) + +LIST_XML = $(LIST_D:.d=.xml) EXTRA_DIST = \ autogen.sh autover.sh valgrind.supp \ @@ -385,8 +400,13 @@ else # HAVE_POSIX CHECKCOUNT_DEFAULT = 500 endif # HAVE_POSIX -CHECKCOUNT = $(if $(filter 1,$(SHORTCHECK)),500,$(CHECKCOUNT_DEFAULT)) -CHECKSIZE = $(if $(filter 1,$(SHORTCHECK)),100,4000) +CHECKCOUNT_0 = $(CHECKCOUNT_DEFAULT) +CHECKCOUNT_1 = 500 +CHECKCOUNT = $(CHECKCOUNT_$(SHORTCHECK)) + +CHECKSIZE_0 = 4000 +CHECKSIZE_1 = 100 +CHECKSIZE = $(CHECKSIZE_$(SHORTCHECK)) CONF = $(srcdir)/test/test-par6.conf THERMAL = $(srcdir)/test/test-par6-thermal.conf From e27d2217936f2cc14d028aa01f991bd7d37426ce Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 22 Oct 2026 18:50:18 +0200 Subject: [PATCH 41/51] Update HISTORY --- HISTORY | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/HISTORY b/HISTORY index 29bf0a47..e413d904 100644 --- a/HISTORY +++ b/HISTORY @@ -1,8 +1,20 @@ SnapRAID HISTORY ================ -15.0 (Work in Progress) -======================= +15.1 2026/10 +============ + * Fix a failed internal inconsistency check when '--gui-touch-before' + is used before the first sync when no content file exists yet. + Also ensure that commands that can save the content file ('touch', + 'scrub', 'rehash', 'rewrite') fail with a clear error message if the + array has not been synced yet. This fix is required by + SnapRAID-Daemon. + * Make test and doc variables in Makefile.am compatible with BSD make, + allowing builds, tests, and documentation generation on BSD systems + without requiring GNU make. + +15.0 2026/10 +============ * Added support for automated filesystem snapshots to ensure data consistency and improve recovery reliability. When the 'snapshot' option is enabled, SnapRAID transparently manages From 7275872c80f4d8ae9d1869fa72f936e7a852512c Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 22 Oct 2026 20:18:52 +0200 Subject: [PATCH 42/51] Pin build-linux runner to ubuntu-24.04 for zfs kernel modules --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed5b2e2d..d60345ab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,7 @@ jobs: - run: make all -j$(sysctl -n hw.ncpu) - run: make dist check build-linux: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v5 - run: | From 4054832c7e385388c6629561bf0ddb16b95698b5 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 22 Oct 2026 21:36:39 +0200 Subject: [PATCH 43/51] Add best-effort Linux disk instance mappings using boot_id and diskseq --- HISTORY | 3 +++ cmdline/unixapp.c | 31 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/HISTORY b/HISTORY index e413d904..dd9a4005 100644 --- a/HISTORY +++ b/HISTORY @@ -3,6 +3,9 @@ SnapRAID HISTORY 15.1 2026/10 ============ + * Add synthetic device IDs on Linux, allowing SnapRAID Daemon to + preserve history for devices without unique hardware IDs while + they remain connected within the same boot. * Fix a failed internal inconsistency check when '--gui-touch-before' is used before the first sync when no content file exists yet. Also ensure that commands that can save the content file ('touch', diff --git a/cmdline/unixapp.c b/cmdline/unixapp.c index 78479c05..8609ba03 100644 --- a/cmdline/unixapp.c +++ b/cmdline/unixapp.c @@ -4447,6 +4447,7 @@ int devmap(void) { #if HAVE_LINUX_DEVICE char path[PATH_MAX]; + char boot_id[64]; DIR* d = opendir("/sys/block"); if (!d) { /* LCOV_EXCL_START */ @@ -4454,6 +4455,22 @@ int devmap(void) /* LCOV_EXCL_STOP */ } + /* optional boot identity: failure must not affect hardware mappings */ + if (sysattr("/proc/sys/kernel/random/boot_id", boot_id, sizeof(boot_id)) != 0 || strlen(boot_id) != 36) { + boot_id[0] = 0; + } else { + for (unsigned i = 0; i < 36; ++i) { + if (i == 8 || i == 13 || i == 18 || i == 23) { + if (boot_id[i] == '-') + continue; + } else if (hexval(boot_id[i]) >= 0) { + continue; + } + boot_id[0] = 0; + break; + } + } + struct dirent* dd; while (1) { char buf[256]; @@ -4517,6 +4534,20 @@ int devmap(void) if (sysattr(path, buf, sizeof(buf)) == 0 && buf[0] != 0) { log_tag("map:/dev/%s:%s\n", dd->d_name, esc_tag(buf)); } + + /* + * This identifies the kernel device instance within one boot, not + * persistent hardware across reconnections. Missing attributes are optional. + */ + if (boot_id[0] != 0) { + pathprint(path, sizeof(path), "/sys/block/%s/diskseq", dd->d_name); + if (sysattr(path, buf, sizeof(buf)) == 0) { + char* end; + unsigned long long diskseq = strtonll(buf, &end, 10); + if (buf[0] >= '0' && buf[0] <= '9' && diskseq != 0 && end != buf && *end == 0) + log_tag("map:/dev/%s:linux-diskseq.%s.%llu\n", esc_tag(dd->d_name), esc_tag(boot_id), diskseq); + } + } } closedir(d); From 5871fd4da356af5c171e6865a6c790b0c37ab6ff Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Fri, 23 Oct 2026 11:05:30 +0200 Subject: [PATCH 44/51] Update CHECKSUMS --- CHECKSUMS | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/CHECKSUMS b/CHECKSUMS index 8d59eba8..80753f42 100644 --- a/CHECKSUMS +++ b/CHECKSUMS @@ -222,17 +222,17 @@ e11c452e2adec376e6fcaee9ff2ec4de4e943c35690229c9c67056c5eaed52a1 snapraid-14.9- 1ec0572359fb31f2827702d792db0bdf6a95161eb9e2a6deff2bdc7e1f5b794b snapraid-14.10-windows-x86.zip 6930d629c73b5307ec32277310ed13887b1bd051e30dc41c75b4752040032d1c snapraid-14.10-x86_64-1_am.txz ceba44e15566386d923d6895ecab81f7437848b03a4cf9585c02d3636197ce19 snapraid-15.0.tar.gz -1e15cc11b26b02fbb54a679c0379161bf3c94aadfaf29081d876990d809a43f9 snapraid-15.0rc2.tar.gz -a546944800a5001c3960715b97ba73cae219c854da1bb55dca16ce70fe06c6d6 snapraid-15.0rc2-1-x86_64.pkg.tar.zst -1327c122a66f91e7780e77431a318907661e690a4aca7fa78f9d504e8dde9e06 snapraid-15.0rc2-1.x86_64.rpm -092d1e91915e7644288805d1a91bd6f20ee0591d70157f5bb5980e26eaf82c1a snapraid-15.0rc2-windows-x64.zip -2c010c4ea69f39d28f266d978d78e0b2c3a845ab47f847cb5a305fc242994691 snapraid-15.0rc2-windows-x86.zip -7ada0dd95bad90895c3faaf627f99da02a4e31ac948d05f012efd2ab05a37d97 snapraid-15.0rc2-x86_64-1_am.txz 9d55d0308ce41beb29956b59f9d2e32d419c930118266d3e08b15f658119f4fb snapraid-15.0-1-x86_64.pkg.tar.zst ad66c0b4c016775c9a759adf2a5202bebb357d319b0063b9cf68befde9a67363 snapraid-15.0-1.x86_64.rpm 704706c21b432719c4da8cd237c1f26327af89c86604dc339a38d63574004a22 snapraid-15.0-windows-x64.zip 58b3ed90307e41b7a6e862d2dc070204c98b5c3ac24718e2fb9fe03e275bc2ce snapraid-15.0-windows-x86.zip 9483df5a39a8b59c32433331ac92a4eef54dcff4722694a5bc5e89bcefb6b3b0 snapraid-15.0-x86_64-1_am.txz +fb2d6a9cf9c8ea30c6962f4a48764dc9012fe61d6e23f40a8d959dadea02278b snapraid-15.1.tar.gz +5e627e08caa8d7647fd40e5288b8d59846d1f3d65d586bae74a57cd99a23f497 snapraid-15.1-1-x86_64.pkg.tar.zst +8922234dd566c876b17cbaadad6a22690d05240250455e504513906d933e16dd snapraid-15.1-1.x86_64.rpm +64849f463986d0d80b948dbb07e9af6402bed230a9cf17dbd1110199aca56486 snapraid-15.1-windows-x64.zip +f7be230f7849d89941a280dc8c2cc6e158844e0fe6f00adbb6aa9ac0f65322b8 snapraid-15.1-windows-x86.zip +4dbd01ab2582a74fab4d7ee8f9b11966d4188fba9eedcd31fcc0a5588cd0c702 snapraid-15.1-x86_64-1_am.txz 128b5b0f80a6a0911e77ed3b5d9d81529f5ea1d8fffbb5112e3af3467d80464a snapraid_14.0-1_amd64.deb b9c41ae2e8edbc413abd3172b7ed2da2cba7fa0e631d0f47fc1dafccf7c6e92c snapraid_14.1-1_amd64.deb e6456b926bb053d34645e31e5e3d8363e952c9869e873eb348abb1149598d331 snapraid_14.2-1_amd64.deb @@ -244,5 +244,5 @@ b71b556da4be6263a904fb8ac489933875467adbb920dd3911e54d87e9637ca3 snapraid_14.6- c0e00a25b9bce40ff74f5a55401c3bfedac1881fd99aaa90a692f5858d417f7d snapraid_14.8-1_amd64.deb 09064689a64d965ac33ee89fb8b2cc14dfec0745f2b3b86a78f377fc353b7e07 snapraid_14.9-1_amd64.deb 15f5fd9f639912c94daae9d75d3567b7a3fc0e18475880f972acd8d8e9e0992c snapraid_14.10-1_amd64.deb -fc875789267ec9b057d14780f04362cc32b3cb4191e707d6e0390f05ff346d46 snapraid_15.0rc2-1_amd64.deb 845091e7eeb2495fc3899dc6e8fab2c3fe7a5945f345c767a7603dab90e2cb84 snapraid_15.0-1_amd64.deb +974cd45fc8f44d5642b49bd9b30e81aca2bcd7689877f9fa73d1e58d3f1b5bda snapraid_15.1-1_amd64.deb From ccf095a6f88f127cb1ef3c798b777e03a7c17163 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 10 Nov 2026 22:27:01 +0100 Subject: [PATCH 45/51] Log OS function failure diagnostics at error level --- os/mingw.c | 106 +++++++++++++++++++------------------- os/unix.c | 148 ++++++++++++++++++++++++++--------------------------- 2 files changed, 127 insertions(+), 127 deletions(-) diff --git a/os/mingw.c b/os/mingw.c index 639f3073..edbf537a 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -1192,8 +1192,8 @@ int windows_fsync(int fd) * So, we use now an extended error reporting. */ windows_errno(error); - os_syslog(OS_LVL_INFO, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); - os_syslog(OS_LVL_INFO, "are you using ATA-over-Ethernet? Please report it."); + os_syslog(OS_LVL_ERROR, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); + os_syslog(OS_LVL_ERROR, "are you using ATA-over-Ethernet? Please report it."); return -1; case ERROR_ACCESS_DENIED : @@ -1249,8 +1249,8 @@ static int windows_vsync(const wchar_t* volume) * So, we use now an extended error reporting. */ windows_errno(error); - os_syslog(OS_LVL_INFO, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); - os_syslog(OS_LVL_INFO, "are you using ATA-over-Ethernet? Please report it."); + os_syslog(OS_LVL_ERROR, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); + os_syslog(OS_LVL_ERROR, "are you using ATA-over-Ethernet? Please report it."); return -1; case ERROR_ACCESS_DENIED : @@ -3206,14 +3206,14 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* create a pipe for the child process's STDOUT */ if (!CreatePipe(&stdout_read_handle, &stdout_write_handle, &sa, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); return -1; } /* ensure the reading handle to the pipe is not inherited */ if (!SetHandleInformation(stdout_read_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdout_write_handle); CloseHandle(stdout_read_handle); return -1; @@ -3222,7 +3222,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch out_f = _open_osfhandle((intptr_t)stdout_read_handle, O_RDONLY | O_BINARY); if (out_f == -1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdout_write_handle); CloseHandle(stdout_read_handle); return -1; @@ -3233,7 +3233,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* create a pipe for the child process's STDERR */ if (!CreatePipe(&stderr_read_handle, &stderr_write_handle, &sa, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); if (has_out) { CloseHandle(stdout_write_handle); close(out_f); @@ -3244,7 +3244,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* ensure the reading handle to the pipe is not inherited */ if (!SetHandleInformation(stderr_read_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stderr_write_handle); CloseHandle(stderr_read_handle); if (has_out) { @@ -3257,7 +3257,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch err_f = _open_osfhandle((intptr_t)stderr_read_handle, O_RDONLY | O_BINARY); if (err_f == -1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stderr_write_handle); CloseHandle(stderr_read_handle); if (has_out) { @@ -3272,7 +3272,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch HANDLE nul_handle = CreateFileW(L"NUL", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, NULL); if (nul_handle == INVALID_HANDLE_VALUE) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to open NUL for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open NUL for spawn, errno=%s(%d)", strerror(errno), errno); if (has_out) { CloseHandle(stdout_write_handle); close(out_f); @@ -3302,7 +3302,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch } pos = argcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { - os_syslog(OS_LVL_INFO, "command to long for spawn"); + os_syslog(OS_LVL_ERROR, "command to long for spawn"); CloseHandle(nul_handle); if (has_out) { CloseHandle(stdout_write_handle); @@ -3322,7 +3322,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch if (process_startup_init(&startup, nul_handle, has_out ? stdout_write_handle : nul_handle, has_err ? stderr_write_handle : nul_handle) != 0) { - os_syslog(OS_LVL_INFO, "failed to create handle list for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create handle list for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(nul_handle); if (has_out) { CloseHandle(stdout_write_handle); @@ -3374,7 +3374,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* validate that the requested user is actually a supported Service Account before attempting logon */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { - os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); + os_syslog(OS_LVL_ERROR, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(nul_handle); @@ -3392,7 +3392,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch const wchar_t* wuser = _stricmp(run_as_user, "LocalService") == 0 ? L"LocalService" : L"NetworkService"; if (!LogonUserW(wuser, L"NT AUTHORITY", NULL, LOGON32_LOGON_SERVICE, LOGON32_PROVIDER_DEFAULT, &h_token)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(nul_handle); @@ -3411,7 +3411,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(h_token); process_startup_done(&startup); @@ -3447,7 +3447,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch process_startup_done(&startup); if (!ret) { windows_errno(create_error); - os_syslog(OS_LVL_INFO, "failed to create process '%s' for spawn, errno=%s(%d)", argv[0], strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create process '%s' for spawn, errno=%s(%d)", argv[0], strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(nul_handle); if (has_out) { @@ -3471,7 +3471,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch if (os_job_handle != 0) { if (!AssignProcessToJobObject(os_job_handle, pi.hProcess)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to assign process to job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to assign process to job object, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3491,7 +3491,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* resume the primary thread now that the process is assigned to the job object */ if (ResumeThread(pi.hThread) == (DWORD)-1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to resume thread for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to resume thread for spawn, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3696,14 +3696,14 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* create pipe for child's STDIN */ if (!CreatePipe(&stdin_read_handle, &stdin_write_handle, &sa, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to create pipe for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create pipe for command, errno=%s(%d)", strerror(errno), errno); return -1; } /* ensure the parent's write end is NOT inherited */ if (!SetHandleInformation(stdin_write_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); return -1; @@ -3720,7 +3720,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t NULL); if (nul == INVALID_HANDLE_VALUE) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to create nul device, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create nul device, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); return -1; @@ -3728,7 +3728,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t ZeroMemory(&pi, sizeof(pi)); if (process_startup_init(&startup, stdin_read_handle, nul, nul) != 0) { - os_syslog(OS_LVL_INFO, "failed to create handle list for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create handle list for command, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); CloseHandle(nul); @@ -3777,7 +3777,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t * Service Account before attempting logon. */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { - os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); + os_syslog(OS_LVL_ERROR, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(stdin_read_handle); @@ -3789,7 +3789,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t const wchar_t* wuser = _stricmp(run_as_user, "LocalService") == 0 ? L"LocalService" : L"NetworkService"; if (!LogonUserW(wuser, L"NT AUTHORITY", NULL, LOGON32_LOGON_SERVICE, LOGON32_PROVIDER_DEFAULT, &h_token)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(stdin_read_handle); @@ -3804,7 +3804,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(stdin_read_handle); @@ -3834,7 +3834,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t process_startup_done(&startup); if (!ret) { windows_errno(create_error); - os_syslog(OS_LVL_INFO, "failed to create process '%s' for command, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create process '%s' for command, errno=%s(%d)", command, strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); @@ -3852,7 +3852,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (os_job_handle != 0) { if (!AssignProcessToJobObject(os_job_handle, pi.hProcess)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to assign command process to job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to assign command process to job object, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3865,7 +3865,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* resume the primary thread now that the process is assigned to the job object */ if (ResumeThread(pi.hThread) == (DWORD)-1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to resume thread for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to resume thread for command, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3888,7 +3888,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (error == ERROR_BROKEN_PIPE || error == ERROR_NO_DATA) break; /* child closed stdin early */ windows_errno(error); - os_syslog(OS_LVL_INFO, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(errno), errno); break; } written += count; @@ -3914,7 +3914,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_INFO, "command %s timeout after %" PRId64 " seconds", command, execution_time); + os_syslog(OS_LVL_ERROR, "command %s timeout after %" PRId64 " seconds", command, execution_time); return -1; } @@ -3926,16 +3926,16 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (exit_code == 0) os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with success", command, execution_time); else - os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); + os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Windows it can happen */ - os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", command, execution_time, (unsigned)status); + os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", command, execution_time, (unsigned)status); return -1; } } @@ -4094,7 +4094,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou /* resolve the script path to prevent symlink attacks */ if (!realpath(script_path, resolved_path)) { - os_syslog(OS_LVL_INFO, "failed to resolve script, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to resolve script, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); return -1; } @@ -4120,9 +4120,9 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou pos = scriptcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { if (errno == EINVAL) - os_syslog(OS_LVL_INFO, "unsupported embedded double quote in script path"); + os_syslog(OS_LVL_ERROR, "unsupported embedded double quote in script path"); else - os_syslog(OS_LVL_INFO, "command to long for script"); + os_syslog(OS_LVL_ERROR, "command to long for script"); return -1; } for (int i = 1; argv[i]; ++i) { @@ -4131,15 +4131,15 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou pos = scriptcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { if (errno == EINVAL) - os_syslog(OS_LVL_INFO, "unsupported embedded double quote in script argument"); + os_syslog(OS_LVL_ERROR, "unsupported embedded double quote in script argument"); else - os_syslog(OS_LVL_INFO, "command to long for script"); + os_syslog(OS_LVL_ERROR, "command to long for script"); return -1; } } pos = fixcat(cmd_buffer, COMMAND_LINE_MAX, pos, L" \""); if (pos < 0) { - os_syslog(OS_LVL_INFO, "command to long for script"); + os_syslog(OS_LVL_ERROR, "command to long for script"); return -1; } cmd_buffer[pos] = 0; @@ -4167,14 +4167,14 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou base_env = GetEnvironmentStringsW(); if (!base_env) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to get environment strings, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to get environment strings, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } combined_env = env_combine(base_env, envp); FreeEnvironmentStringsW(base_env); if (!combined_env) { - os_syslog(OS_LVL_INFO, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } @@ -4205,7 +4205,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou * Service Account before attempting logon. */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { - os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); + os_syslog(OS_LVL_ERROR, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); return -1; } @@ -4213,7 +4213,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou const wchar_t* wuser = _stricmp(run_as_user, "LocalService") == 0 ? L"LocalService" : L"NetworkService"; if (!LogonUserW(wuser, L"NT AUTHORITY", NULL, LOGON32_LOGON_SERVICE, LOGON32_PROVIDER_DEFAULT, &h_token)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } @@ -4224,7 +4224,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(h_token); return -1; @@ -4234,7 +4234,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (envp != NULL) { combined_env = env_combine((const WCHAR*)env, envp); if (!combined_env) { - os_syslog(OS_LVL_INFO, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); DestroyEnvironmentBlock(env); CloseHandle(h_token); @@ -4261,7 +4261,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou } if (!ret) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to create process '%s' for script, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create process '%s' for script, errno=%s(%d)", resolved_path, strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } @@ -4270,7 +4270,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (os_job_handle != 0) { if (!AssignProcessToJobObject(os_job_handle, pi.hProcess)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to assign script process to job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to assign script process to job object, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -4282,7 +4282,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou /* resume the primary thread now that the process is assigned to the job object */ if (ResumeThread(pi.hThread) == (DWORD)-1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_INFO, "failed to resume thread for script, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to resume thread for script, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -4309,7 +4309,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_INFO, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); + os_syslog(OS_LVL_ERROR, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); return -1; } @@ -4321,16 +4321,16 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (exit_code == 0) os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with success", resolved_path, execution_time); else - os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); + os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Windows it can happen */ - os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", resolved_path, execution_time, (unsigned)status); + os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", resolved_path, execution_time, (unsigned)status); return -1; } } diff --git a/os/unix.c b/os/unix.c index 9c4bcbf0..6ba1eee6 100644 --- a/os/unix.c +++ b/os/unix.c @@ -461,24 +461,24 @@ static int verify_shebang_interpreter(int fd, const char* script_path) bytes_read = pread(fd, shebang, sizeof(shebang) - 1, 0); /* reserve space for the terminating 0 */ if (bytes_read < 0) { - os_syslog(OS_LVL_INFO, "failed to read script shebang, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to read script shebang, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); return -1; } if (bytes_read < 4) { - os_syslog(OS_LVL_INFO, "script %s is too small or missing a shebang", script_path); + os_syslog(OS_LVL_ERROR, "script %s is too small or missing a shebang", script_path); return -1; } shebang[bytes_read] = 0; /* check for shebang */ if (shebang[0] != '#' || shebang[1] != '!') { - os_syslog(OS_LVL_INFO, "script %s is missing shebang (#!)", script_path); + os_syslog(OS_LVL_ERROR, "script %s is missing shebang (#!)", script_path); return -1; } char* end_of_line = strchr(shebang, '\n'); if (!end_of_line || end_of_line - shebang > 128) { - os_syslog(OS_LVL_INFO, "script %s has invalid or overlong shebang (#!), exceeds 126 characters", script_path); + os_syslog(OS_LVL_ERROR, "script %s has invalid or overlong shebang (#!), exceeds 126 characters", script_path); return -1; } *end_of_line = 0; @@ -489,7 +489,7 @@ static int verify_shebang_interpreter(int fd, const char* script_path) ++interpreter; if (*interpreter == 0) { - os_syslog(OS_LVL_INFO, "script %s has empty shebang", script_path); + os_syslog(OS_LVL_ERROR, "script %s has empty shebang", script_path); return -1; } @@ -510,49 +510,49 @@ static int verify_shebang_interpreter(int fd, const char* script_path) } if (!found) { - os_syslog(OS_LVL_INFO, "script %s uses disallowed interpreter %s", script_path, interpreter); + os_syslog(OS_LVL_ERROR, "script %s uses disallowed interpreter %s", script_path, interpreter); return -1; } /* verify interpreter exists and is safe */ if (stat(interpreter, &st) != 0) { - os_syslog(OS_LVL_INFO, "interpreter %s does not exist, errno=%s(%d)", interpreter, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "interpreter %s does not exist, errno=%s(%d)", interpreter, strerror(errno), errno); return -1; } /* interpreter must be a regular file */ if (!S_ISREG(st.st_mode)) { - os_syslog(OS_LVL_INFO, "interpreter %s must be a regular file", interpreter); + os_syslog(OS_LVL_ERROR, "interpreter %s must be a regular file", interpreter); return -1; } /* interpreter must be owned by root */ if (st.st_uid != 0) { - os_syslog(OS_LVL_INFO, "interpreter %s not owned by root", interpreter); + os_syslog(OS_LVL_ERROR, "interpreter %s not owned by root", interpreter); return -1; } /* interpreter must be not world-writable */ if (st.st_mode & S_IWOTH) { - os_syslog(OS_LVL_INFO, "interpreter %s is world-writable", interpreter); + os_syslog(OS_LVL_ERROR, "interpreter %s is world-writable", interpreter); return -1; } /* interpreter must be not group-writable (unless group is root) */ if ((st.st_mode & S_IWGRP) && st.st_gid != 0) { - os_syslog(OS_LVL_INFO, "interpreter %s is group-writable by non-root group", interpreter); + os_syslog(OS_LVL_ERROR, "interpreter %s is group-writable by non-root group", interpreter); return -1; } /* interpreter must be executable */ if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { - os_syslog(OS_LVL_INFO, "interpreter %s is not executable", interpreter); + os_syslog(OS_LVL_ERROR, "interpreter %s is not executable", interpreter); return -1; } /* interpreter must be not setuid / setgid */ if (st.st_mode & (S_ISUID | S_ISGID)) { - os_syslog(OS_LVL_INFO, "file %s has setuid/setgid bits set", interpreter); + os_syslog(OS_LVL_ERROR, "file %s has setuid/setgid bits set", interpreter); return -1; } @@ -595,7 +595,7 @@ static int verify_shebang_interpreter(int fd, const char* script_path) * * Returns an open file descriptor (>= 0) on success. The caller is * responsible for closing it. Returns -1 on any verification failure; - * the specific reason is emitted via os_syslog(OS_LVL_INFO, ...). + * the specific reason is emitted via os_syslog(OS_LVL_ERROR, ...). */ static int verify_executable(const char* exec_path, char* resolved_path, int is_script) { @@ -610,29 +610,29 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ /* verify path is absolute */ if (exec_path[0] != '/') { - os_syslog(OS_LVL_INFO, "path %s must be absolute", exec_path); + os_syslog(OS_LVL_ERROR, "path %s must be absolute", exec_path); return -1; } /* resolve the path to prevent symlink attacks */ if (!realpath(exec_path, resolved_path)) { - os_syslog(OS_LVL_INFO, "failed to resolve %s, errno=%s(%d)", exec_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to resolve %s, errno=%s(%d)", exec_path, strerror(errno), errno); return -1; } char* last_slash = strrchr(resolved_path, '/'); if (last_slash == 0) { - os_syslog(OS_LVL_INFO, "relative execution of %s not allowed", resolved_path); + os_syslog(OS_LVL_ERROR, "relative execution of %s not allowed", resolved_path); return -1; } if (last_slash == resolved_path) { - os_syslog(OS_LVL_INFO, "root dir execution of %s not allowed", resolved_path); + os_syslog(OS_LVL_ERROR, "root dir execution of %s not allowed", resolved_path); return -1; } const char* exec_name = last_slash + 1; if (exec_name[0] == 0) { - os_syslog(OS_LVL_INFO, "no executable name in %s", exec_path); + os_syslog(OS_LVL_ERROR, "no executable name in %s", exec_path); return -1; } @@ -643,33 +643,33 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ int dir_fd = open(dir_path, O_PATH | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (dir_fd < 0) { - os_syslog(OS_LVL_INFO, "failed to open directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); return -1; } if (fstat(dir_fd, &st) != 0) { - os_syslog(OS_LVL_INFO, "failed to stat directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to stat directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); close(dir_fd); return -1; } /* directory must be owned by root or the daemon's real user */ if (st.st_uid != process_uid && st.st_uid != process_euid && st.st_uid != 0) { - os_syslog(OS_LVL_INFO, "directory %s owner must match the daemon owner or be root", dir_path); + os_syslog(OS_LVL_ERROR, "directory %s owner must match the daemon owner or be root", dir_path); close(dir_fd); return -1; } /* directory must be not group-writable unless group matches daemon */ if ((st.st_mode & S_IWGRP) && st.st_gid != process_gid && st.st_gid != process_egid && st.st_gid != 0) { - os_syslog(OS_LVL_INFO, "directory %s must be not group-writable unless group matches daemon owner or root", dir_path); + os_syslog(OS_LVL_ERROR, "directory %s must be not group-writable unless group matches daemon owner or root", dir_path); close(dir_fd); return -1; } /* directory must be not world-writable */ if (st.st_mode & S_IWOTH) { - os_syslog(OS_LVL_INFO, "directory %s must be not world-writable", dir_path); + os_syslog(OS_LVL_ERROR, "directory %s must be not world-writable", dir_path); close(dir_fd); return -1; } @@ -689,7 +689,7 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ int fd = openat(dir_fd, exec_name, flags); if (fd < 0) { - os_syslog(OS_LVL_INFO, "failed to open %s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open %s, errno=%s(%d)", resolved_path, strerror(errno), errno); close(dir_fd); return -1; } @@ -698,14 +698,14 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ /* get the file handle (TOCTOU Protection) */ if (fstat(fd, &st) == -1) { - os_syslog(OS_LVL_INFO, "failed to stat %s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to stat %s, errno=%s(%d)", resolved_path, strerror(errno), errno); close(fd); return -1; } /* ensure it's a regular file */ if (!S_ISREG(st.st_mode)) { - os_syslog(OS_LVL_INFO, "file %s is not a regular file", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s is not a regular file", resolved_path); close(fd); return -1; } @@ -714,7 +714,7 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ if (!is_script) { char magic[2]; if (read(fd, magic, 2) == 2 && magic[0] == '#' && magic[1] == '!') { - os_syslog(OS_LVL_INFO, "file %s is a script, which is not supported", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s is a script, which is not supported", resolved_path); close(fd); return -1; } @@ -725,35 +725,35 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ /* ensure it has execute permissions */ if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { - os_syslog(OS_LVL_INFO, "file %s is not an executable", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s is not an executable", resolved_path); close(fd); return -1; } /* must be owned by root or the daemon's real user */ if (st.st_uid != process_uid && st.st_uid != process_euid && st.st_uid != 0) { - os_syslog(OS_LVL_INFO, "file %s owner must match the daemon owner or be root", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s owner must match the daemon owner or be root", resolved_path); close(fd); return -1; } /* must be not group-writable unless group matches daemon */ if ((st.st_mode & S_IWGRP) && st.st_gid != process_gid && st.st_gid != process_egid && st.st_gid != 0) { - os_syslog(OS_LVL_INFO, "file %s must be not group-writable unless group matches daemon owner or root", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s must be not group-writable unless group matches daemon owner or root", resolved_path); close(fd); return -1; } /* must be not world-writable */ if (st.st_mode & S_IWOTH) { - os_syslog(OS_LVL_INFO, "file %s must be not world-writable", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s must be not world-writable", resolved_path); close(fd); return -1; } /* must be not setuid / setgid */ if (st.st_mode & (S_ISUID | S_ISGID)) { - os_syslog(OS_LVL_INFO, "file %s has setuid/setgid bits set", resolved_path); + os_syslog(OS_LVL_ERROR, "file %s has setuid/setgid bits set", resolved_path); close(fd); return -1; } @@ -765,13 +765,13 @@ int os_validate_exec_input(const char* str) { /* reject paths trying to go up levels */ if (strstr(str, "..") != 0) { - os_syslog(OS_LVL_INFO, "invalid argument %s for execution", str); + os_syslog(OS_LVL_ERROR, "invalid argument %s for execution", str); return -1; } /* reject inputs starting with '-' to prevent Flag Injection */ if (str[0] == '-') { - os_syslog(OS_LVL_INFO, "invalid argument %s for execution", str); + os_syslog(OS_LVL_ERROR, "invalid argument %s for execution", str); return -1; } @@ -810,7 +810,7 @@ static int os_user_lookup(const char* run_as_user, struct os_user* user) char pw_buf[PASSWD_MAX]; int err = getpwnam_r(run_as_user, &pw, pw_buf, sizeof(pw_buf), &pw_res); if (err != 0 || !pw_res) { - os_syslog(OS_LVL_INFO, "failed to get user info for %s, errno=%s(%d)", run_as_user, strerror(err ? err : ENOENT), err ? err : ENOENT); + os_syslog(OS_LVL_ERROR, "failed to get user info for %s, errno=%s(%d)", run_as_user, strerror(err ? err : ENOENT), err ? err : ENOENT); return -1; } @@ -822,7 +822,7 @@ static int os_user_lookup(const char* run_as_user, struct os_user* user) int ngroups = sizeof(groups_buf) / sizeof(groups_buf[0]); if (getgrouplist(pw.pw_name, (os_group_t)pw.pw_gid, groups_buf, &ngroups) < 0) { - os_syslog(OS_LVL_INFO, "failed to get group list for %s, exceeds maximum of %d groups", run_as_user, GROUPS_MAX); + os_syslog(OS_LVL_ERROR, "failed to get group list for %s, exceeds maximum of %d groups", run_as_user, GROUPS_MAX); return -1; } @@ -1198,7 +1198,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou pid = fork(); if (pid < 0) { - os_syslog(OS_LVL_INFO, "failed to fork script=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to fork script=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); pid_unpublish(pid_slot, 0); close(fd); return -1; @@ -1323,7 +1323,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou } if (ret == -1) { - os_syslog(OS_LVL_INFO, "failed to wait for script, path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to wait for script, path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); return -1; } @@ -1331,7 +1331,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_INFO, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); + os_syslog(OS_LVL_ERROR, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); return 128 + SIGALRM; } @@ -1344,17 +1344,17 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (exit_code == 0) os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with success", resolved_path, execution_time); else - os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); + os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Linux it should never happen */ - os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds for unknown reason, status=%d", resolved_path, execution_time, status); + os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds for unknown reason, status=%d", resolved_path, execution_time, status); return -1; } } @@ -1373,14 +1373,14 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* create pipe only if we have text to send */ if (stdin_text != 0) { if (pipe_cloexec(pipe_fds) < 0) { - os_syslog(OS_LVL_INFO, "failed to create input pipe, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to create input pipe, errno=%s(%d)", strerror(errno), errno); return -1; } int flags = fcntl(pipe_fds[1], F_GETFL); if (flags < 0 || fcntl(pipe_fds[1], F_SETFL, flags | O_NONBLOCK) < 0) { - os_syslog(OS_LVL_INFO, "failed to make input pipe non-blocking, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to make input pipe non-blocking, errno=%s(%d)", strerror(errno), errno); close(pipe_fds[0]); close(pipe_fds[1]); return -1; @@ -1411,7 +1411,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t pid = fork(); if (pid < 0) { - os_syslog(OS_LVL_INFO, "failed to fork command=%s, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to fork command=%s, errno=%s(%d)", command, strerror(errno), errno); pid_unpublish(pid_slot, 0); if (pipe_fds[0] != -1) { close(pipe_fds[0]); @@ -1509,7 +1509,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t pid_unpublish(pid_slot, pid); if (input_error != 0) - os_syslog(OS_LVL_INFO, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(input_error), input_error); + os_syslog(OS_LVL_ERROR, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(input_error), input_error); if (ret != -1) { do { @@ -1518,7 +1518,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t } if (ret == -1) { - os_syslog(OS_LVL_INFO, "failed to wait for command, command=%s, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to wait for command, command=%s, errno=%s(%d)", command, strerror(errno), errno); return -1; } @@ -1526,7 +1526,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_INFO, "command %s timeout after %" PRId64 " seconds", command, execution_time); + os_syslog(OS_LVL_ERROR, "command %s timeout after %" PRId64 " seconds", command, execution_time); return 128 + SIGALRM; } @@ -1538,16 +1538,16 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (exit_code == 0) os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with success", command, execution_time); else - os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); + os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Linux it should never happen */ - os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds for unknown reason, status=%d", command, execution_time, status); + os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds for unknown reason, status=%d", command, execution_time, status); return -1; } } @@ -1584,7 +1584,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char if (has_out) { if (pipe_cloexec(out_pipe) < 0) { - os_syslog(OS_LVL_INFO, "failed to open output pipe, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open output pipe, errno=%s(%d)", strerror(errno), errno); close(fd); return -1; } @@ -1592,7 +1592,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char if (has_err) { if (pipe_cloexec(err_pipe) < 0) { - os_syslog(OS_LVL_INFO, "failed to open error pipe, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to open error pipe, errno=%s(%d)", strerror(errno), errno); if (has_out) { close(out_pipe[0]); close(out_pipe[1]); @@ -1634,7 +1634,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char pid = fork(); if (pid < 0) { - os_syslog(OS_LVL_INFO, "failed to fork path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to fork path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); pid_unpublish(pid_slot, 0); if (has_out) { close(out_pipe[0]); @@ -1819,7 +1819,7 @@ pid_t os_wait(pid_t pid, int* status, pid_t* pid_slot) } while (ret == -1 && errno == EINTR); if (ret == -1) { - os_syslog(OS_LVL_INFO, "failed to wait, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to wait, errno=%s(%d)", strerror(errno), errno); } return ret; @@ -1996,7 +1996,7 @@ int os_privileges_drop(void) /* find the unprivileged user "nobody" */ struct passwd* pw = getpwnam("nobody"); if (!pw) { - os_syslog(OS_LVL_INFO, "unprivileged user 'nobody' not found"); + os_syslog(OS_LVL_ERROR, "unprivileged user 'nobody' not found"); return -1; } @@ -2010,7 +2010,7 @@ int os_privileges_drop(void) */ privileged_group_count = getgroups(0, 0); if (privileged_group_count < 0) { - os_syslog(OS_LVL_INFO, "failed to get supplementary group count, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to get supplementary group count, errno=%s(%d)", strerror(errno), errno); return -1; } @@ -2022,12 +2022,12 @@ int os_privileges_drop(void) } if (getgroups(privileged_group_count, privileged_groups) != privileged_group_count) { - os_syslog(OS_LVL_INFO, "failed to get supplementary groups, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to get supplementary groups, errno=%s(%d)", strerror(errno), errno); return -1; } if (setgroups(0, 0) != 0) { - os_syslog(OS_LVL_INFO, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); return -1; } @@ -2035,14 +2035,14 @@ int os_privileges_drop(void) } if (setegid(unpriv_gid) != 0) { - os_syslog(OS_LVL_INFO, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); return -1; } gid_dropped = 1; if (seteuid(unpriv_uid) != 0) { - os_syslog(OS_LVL_INFO, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); return -1; } @@ -2072,7 +2072,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2083,7 +2083,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2094,7 +2094,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2105,7 +2105,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2116,7 +2116,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2127,7 +2127,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2147,7 +2147,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2158,7 +2158,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2170,7 +2170,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2181,7 +2181,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2192,7 +2192,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2204,7 +2204,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_INFO, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } From 029c3940fe356324af5bd1ef0181a4f2a7893c00 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 10 Nov 2026 22:49:46 +0100 Subject: [PATCH 46/51] Avoid creating a redundant Windows Job Object when already running inside an existing job --- os/mingw.c | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/os/mingw.c b/os/mingw.c index edbf537a..4025d3ef 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -4661,21 +4661,33 @@ void os_init(unsigned opt) os_exit(); } - /* initialize job object to terminate child processes if daemon dies */ - os_job_handle = CreateJobObjectW(0, 0); - if (os_job_handle != 0) { - JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; - ZeroMemory(&info, sizeof(info)); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if (!SetInformationJobObject(os_job_handle, JobObjectExtendedLimitInformation, &info, sizeof(info))) { - windows_errno(GetLastError()); - os_syslog(OS_LVL_WARNING, "failed to configure job object, errno=%s(%d)", strerror(errno), errno); - CloseHandle(os_job_handle); - os_job_handle = 0; - } + /* + * Skip creating a separate job object if already running inside one + * (e.g. spawned by snapraid-daemon, a service, or Task Scheduler). + * Child processes automatically inherit the parent's job object, and + * attempting to assign them to a second independent job object would + * fail with ERROR_ACCESS_DENIED. + */ + BOOL in_job = 0; + if (IsProcessInJob(GetCurrentProcess(), 0, &in_job) && in_job) { + os_job_handle = 0; } else { - windows_errno(GetLastError()); - os_syslog(OS_LVL_WARNING, "failed to create job object, errno=%s(%d)", strerror(errno), errno); + /* initialize job object to terminate child processes if daemon dies */ + os_job_handle = CreateJobObjectW(0, 0); + if (os_job_handle != 0) { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; + ZeroMemory(&info, sizeof(info)); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!SetInformationJobObject(os_job_handle, JobObjectExtendedLimitInformation, &info, sizeof(info))) { + windows_errno(GetLastError()); + os_syslog(OS_LVL_WARNING, "failed to configure job object, errno=%s(%d)", strerror(errno), errno); + CloseHandle(os_job_handle); + os_job_handle = 0; + } + } else { + windows_errno(GetLastError()); + os_syslog(OS_LVL_WARNING, "failed to create job object, errno=%s(%d)", strerror(errno), errno); + } } /* check for Wine presence */ From 83f3bee5d0df72f42d9ca76cc475fb94bcfea676 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 10 Nov 2026 23:18:42 +0100 Subject: [PATCH 47/51] Fix typos in Windows error messages --- os/mingw.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/os/mingw.c b/os/mingw.c index 4025d3ef..955fbfd5 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -3213,7 +3213,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* ensure the reading handle to the pipe is not inherited */ if (!SetHandleInformation(stdout_read_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to set handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdout_write_handle); CloseHandle(stdout_read_handle); return -1; @@ -3244,7 +3244,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* ensure the reading handle to the pipe is not inherited */ if (!SetHandleInformation(stderr_read_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to set handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stderr_write_handle); CloseHandle(stderr_read_handle); if (has_out) { @@ -3302,7 +3302,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch } pos = argcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { - os_syslog(OS_LVL_ERROR, "command to long for spawn"); + os_syslog(OS_LVL_ERROR, "command too long for spawn"); CloseHandle(nul_handle); if (has_out) { CloseHandle(stdout_write_handle); @@ -3703,7 +3703,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* ensure the parent's write end is NOT inherited */ if (!SetHandleInformation(stdin_write_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_ERROR, "failed to set handle information for command, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); return -1; @@ -4122,7 +4122,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (errno == EINVAL) os_syslog(OS_LVL_ERROR, "unsupported embedded double quote in script path"); else - os_syslog(OS_LVL_ERROR, "command to long for script"); + os_syslog(OS_LVL_ERROR, "command too long for script"); return -1; } for (int i = 1; argv[i]; ++i) { @@ -4133,13 +4133,13 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (errno == EINVAL) os_syslog(OS_LVL_ERROR, "unsupported embedded double quote in script argument"); else - os_syslog(OS_LVL_ERROR, "command to long for script"); + os_syslog(OS_LVL_ERROR, "command too long for script"); return -1; } } pos = fixcat(cmd_buffer, COMMAND_LINE_MAX, pos, L" \""); if (pos < 0) { - os_syslog(OS_LVL_ERROR, "command to long for script"); + os_syslog(OS_LVL_ERROR, "command too long for script"); return -1; } cmd_buffer[pos] = 0; From 4e8ed5cdbd5ed7b42bd206636c49e8521d770111 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 10 Nov 2026 23:21:15 +0100 Subject: [PATCH 48/51] Update HISTORY --- HISTORY | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/HISTORY b/HISTORY index dd9a4005..88e2cfa3 100644 --- a/HISTORY +++ b/HISTORY @@ -1,6 +1,14 @@ SnapRAID HISTORY ================ +15.2 2026/11 +============ + * Avoid creating a redundant Windows Job Object when SnapRAID is + already running inside an existing job (such as under SnapRAID + Daemon). This prevents potential execution failures of child + processes (such as smartctl) on systems where nested job objects are + not supported. + 15.1 2026/10 ============ * Add synthetic device IDs on Linux, allowing SnapRAID Daemon to From 487305a760978e79e92c55426eb4999ae44f9ac0 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 10 Nov 2026 23:40:25 +0100 Subject: [PATCH 49/51] Use ubuntu-26.04 build server --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d60345ab..f98b681f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,16 +20,16 @@ jobs: - run: make all -j$(sysctl -n hw.ncpu) - run: make dist check build-linux: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 steps: - uses: actions/checkout@v5 - run: | sudo apt-get update sudo apt-get install -y \ - linux-modules-extra-$(uname -r) \ + linux-main-modules-zfs-$(uname -r) \ zfsutils-linux sudo modprobe zfs - zfs version + sudo zfs version - run: autoreconf -i - run: ./configure --enable-warning-as-error - run: make all -j$(nproc) From 47e439ddffa41d3f998174d7a5bc416a290ee3cc Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 11 Nov 2026 21:58:10 +0100 Subject: [PATCH 50/51] Skip devices without a backing block device (network/virtual filesystems) in device operations --- HISTORY | 5 +++++ cmdline/device.c | 7 +++++-- cmdline/unixapp.c | 13 +++++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/HISTORY b/HISTORY index 88e2cfa3..4e718e0e 100644 --- a/HISTORY +++ b/HISTORY @@ -8,6 +8,11 @@ SnapRAID HISTORY Daemon). This prevents potential execution failures of child processes (such as smartctl) on systems where nested job objects are not supported. + * Skip disks on network or virtual filesystems (such as NFS, CIFS, + FUSE, or tmpfs) in device operations like 'up', 'down', 'smart', + and 'probe' on Linux, instead of failing with an error. If no + physical device remains, the operation completes successfully with + an informational message. 15.1 2026/10 ============ diff --git a/cmdline/device.c b/cmdline/device.c index 759eb649..2d494281 100644 --- a/cmdline/device.c +++ b/cmdline/device.c @@ -1237,13 +1237,16 @@ int state_device_force(struct snapraid_state* state, int operation, tommy_list* } } - /* distinguish skipped devices from unsupported device queries */ if (ret == 0 && tommy_list_empty(&low)) { - ret = -1; if (tommy_list_empty(&high)) { + ret = -1; log_error(ESOFT, "No accessible devices selected for this operation.\n"); goto bail; } + + /* not an error, the selected disks are on network or virtual filesystems without a block device to manage */ + msg_info("No physical devices found for the selected disks, nothing to do.\n"); + goto bail; } if (ret != 0) { diff --git a/cmdline/unixapp.c b/cmdline/unixapp.c index 8609ba03..0e23e0de 100644 --- a/cmdline/unixapp.c +++ b/cmdline/unixapp.c @@ -4375,6 +4375,16 @@ int devquery(tommy_list* high, tommy_list* low) struct dev_struct* dev = j->data; uint64_t access_stat; + /* + * Major 0 is an anonymous superblock (NFS, CIFS, FUSE, tmpfs, overlay, ...) + * not backed by a block device. Btrfs/Bcachefs/ZFS are already dereferenced + * to their member devices, so what remains has no physical disk to manage. + */ + if (major(dev->device) == 0) { + log_tag("device:%s:skip:%u:%u: no block device\n", esc_tag(devinfo->name), major(dev->device), minor(dev->device)); + continue; + } + /* retrieve access stat for the high level device */ if (devstat(dev->device, &access_stat) == 0) { /* cumulate access stat in the first split */ @@ -4420,6 +4430,9 @@ int devquery(tommy_list* high, tommy_list* low) #else (void)high; (void)low; + + /* device operations are unsupported in this platform */ + failed = 1; #endif return failed ? -1 : degraded ? 1 : 0; From 8350e7bb0f84894961020b58870ea86e4e1a200a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 16 Nov 2026 09:44:22 +0100 Subject: [PATCH 51/51] Update CHECKSUMS --- CHECKSUMS | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHECKSUMS b/CHECKSUMS index 80753f42..9f38dda3 100644 --- a/CHECKSUMS +++ b/CHECKSUMS @@ -233,6 +233,12 @@ fb2d6a9cf9c8ea30c6962f4a48764dc9012fe61d6e23f40a8d959dadea02278b snapraid-15.1. 64849f463986d0d80b948dbb07e9af6402bed230a9cf17dbd1110199aca56486 snapraid-15.1-windows-x64.zip f7be230f7849d89941a280dc8c2cc6e158844e0fe6f00adbb6aa9ac0f65322b8 snapraid-15.1-windows-x86.zip 4dbd01ab2582a74fab4d7ee8f9b11966d4188fba9eedcd31fcc0a5588cd0c702 snapraid-15.1-x86_64-1_am.txz +0740169f66c506c29846a5aef57d2b11ffe5c354c5bb0ba260f2b5f328f985c5 snapraid-15.2.tar.gz +208a5d99c94889e2357f0211d716bababcfb8289907167220563f734fb936e96 snapraid-15.2-1-x86_64.pkg.tar.zst +f1c37fa43beea8445858520a5cd514198d8c0c0ac5a5ab588adcd02e872ee681 snapraid-15.2-1.x86_64.rpm +882334e5102b418ace0753f9335a99783fc784e149429c29911247b56f04e438 snapraid-15.2-windows-x64.zip +942157645a532e2a280ca404625476c42591436f2292491b1faedb9d48efd07e snapraid-15.2-windows-x86.zip +7bb455cb77586dd2d90f6868c66dd9cdf6774aa89c099f1026a851407ac7adac snapraid-15.2-x86_64-1_am.txz 128b5b0f80a6a0911e77ed3b5d9d81529f5ea1d8fffbb5112e3af3467d80464a snapraid_14.0-1_amd64.deb b9c41ae2e8edbc413abd3172b7ed2da2cba7fa0e631d0f47fc1dafccf7c6e92c snapraid_14.1-1_amd64.deb e6456b926bb053d34645e31e5e3d8363e952c9869e873eb348abb1149598d331 snapraid_14.2-1_amd64.deb @@ -246,3 +252,4 @@ c0e00a25b9bce40ff74f5a55401c3bfedac1881fd99aaa90a692f5858d417f7d snapraid_14.8- 15f5fd9f639912c94daae9d75d3567b7a3fc0e18475880f972acd8d8e9e0992c snapraid_14.10-1_amd64.deb 845091e7eeb2495fc3899dc6e8fab2c3fe7a5945f345c767a7603dab90e2cb84 snapraid_15.0-1_amd64.deb 974cd45fc8f44d5642b49bd9b30e81aca2bcd7689877f9fa73d1e58d3f1b5bda snapraid_15.1-1_amd64.deb +9c78f158347cc0340ed5c9877724bb4f3b5e5fb48b43332ee7e0e7b6a63642d9 snapraid_15.2-1_amd64.deb