diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f98b681f..a8c2d41a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,16 +20,16 @@ jobs: - run: make all -j$(sysctl -n hw.ncpu) - run: make dist check build-linux: - runs-on: ubuntu-26.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v5 - run: | sudo apt-get update sudo apt-get install -y \ - linux-main-modules-zfs-$(uname -r) \ + linux-modules-extra-$(uname -r) \ zfsutils-linux sudo modprobe zfs - sudo zfs version + zfs version - run: autoreconf -i - run: ./configure --enable-warning-as-error - run: make all -j$(nproc) @@ -90,12 +90,12 @@ jobs: with: usesh: true prepare: | - pkg install -y autoconf automake + pkg install -y autoconf automake gmake run: | autoreconf -i - ./configure --enable-warning-as-error - make all -j$(sysctl -n hw.ncpu) - make dist check + ./configure --enable-warning-as-error MAKE=gmake + gmake all -j$(sysctl -n hw.ncpu) + gmake dist check build-musl: runs-on: ubuntu-latest container: alpine:latest diff --git a/AGENTS.md b/AGENTS.md index bc96d77a..c6e0753e 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,9 +105,9 @@ SnapRAID is designed to recover from an interruption, including `SIGKILL`, at an - **Format**: Enforce via `uncrustify -c uncrustify.cfg --no-backup *.c *.h` (or `uncrustify -c linux.cfg --no-backup *.c *.h` for `raid/` directory and subdirectories) - **Naming**: Snake_case for functions, UPPER_CASE for macros/constants - **Indentation**: Tabs for indentation, no alignment (existing codebase style) -- **Comments**: Use `/** ... */` for multiline top comments preceding declarations and `/* ... */` for multiline comments within code. Use `/* first letter lowercase */` for single-line inline notes. +- **Comments**: C-style `/** */` for multiline comments; C `/* first letter lowercase */` for single-line inline notes - **Critical Comments**: Always add a comment at non-obvious critical points, especially around data-integrity invariants, crash recovery, fallback behavior, concurrency, and fatal versus best-effort error handling. Explain why the logic is required, not merely what the code does. -- **Headers**: All `.h` files have include guards (`#ifndef NAME_H`) +- **Headers**: All `.h` files have include guards (`#ifndef __NAME_H`) - **Preferences**: Use 0 instead of NULL and '\0' - **Preferences**: Use prefix ++variable and --variable instead of postfix variable++ and variable-- where both are equivalent - **Safety Checks**: Avoid adding safety checks for conditions that never happen diff --git a/CHECKSUMS b/CHECKSUMS index 9f38dda3..b13f1562 100644 --- a/CHECKSUMS +++ b/CHECKSUMS @@ -215,30 +215,6 @@ e11c452e2adec376e6fcaee9ff2ec4de4e943c35690229c9c67056c5eaed52a1 snapraid-14.9- 84af790cd25c8f2d773386bca7320cb5cf81706f850eb1fda33d4b9ff4e0d692 snapraid-14.9-windows-x64.zip 7ec39de0cf7cabd09061bfc54848084b4505e77b27556850fc53cb5f23080003 snapraid-14.9-windows-x86.zip 40cee688cb1a322810eaac4e3655313c0075b791e18783bec2588084a9e47f09 snapraid-14.9-x86_64-1_am.txz -27be09151c8e779019da7d9e0501b03f56ae83da489cdaf5cd85463075cec9a2 snapraid-14.10.tar.gz -6f5a31ee04649c9b2b7b0612619f401f160c4f1a4a56947ff50381cc6036b8cc snapraid-14.10-1-x86_64.pkg.tar.zst -461778095cd40e9ef1231999dd4924499c1f6cab23e233ab811f37e29b81aefe snapraid-14.10-1.x86_64.rpm -6e8892175bfe240ae7828d513342180a249511a2d5baf8c003ae8b2bb82e0f46 snapraid-14.10-windows-x64.zip -1ec0572359fb31f2827702d792db0bdf6a95161eb9e2a6deff2bdc7e1f5b794b snapraid-14.10-windows-x86.zip -6930d629c73b5307ec32277310ed13887b1bd051e30dc41c75b4752040032d1c snapraid-14.10-x86_64-1_am.txz -ceba44e15566386d923d6895ecab81f7437848b03a4cf9585c02d3636197ce19 snapraid-15.0.tar.gz -9d55d0308ce41beb29956b59f9d2e32d419c930118266d3e08b15f658119f4fb snapraid-15.0-1-x86_64.pkg.tar.zst -ad66c0b4c016775c9a759adf2a5202bebb357d319b0063b9cf68befde9a67363 snapraid-15.0-1.x86_64.rpm -704706c21b432719c4da8cd237c1f26327af89c86604dc339a38d63574004a22 snapraid-15.0-windows-x64.zip -58b3ed90307e41b7a6e862d2dc070204c98b5c3ac24718e2fb9fe03e275bc2ce snapraid-15.0-windows-x86.zip -9483df5a39a8b59c32433331ac92a4eef54dcff4722694a5bc5e89bcefb6b3b0 snapraid-15.0-x86_64-1_am.txz -fb2d6a9cf9c8ea30c6962f4a48764dc9012fe61d6e23f40a8d959dadea02278b snapraid-15.1.tar.gz -5e627e08caa8d7647fd40e5288b8d59846d1f3d65d586bae74a57cd99a23f497 snapraid-15.1-1-x86_64.pkg.tar.zst -8922234dd566c876b17cbaadad6a22690d05240250455e504513906d933e16dd snapraid-15.1-1.x86_64.rpm -64849f463986d0d80b948dbb07e9af6402bed230a9cf17dbd1110199aca56486 snapraid-15.1-windows-x64.zip -f7be230f7849d89941a280dc8c2cc6e158844e0fe6f00adbb6aa9ac0f65322b8 snapraid-15.1-windows-x86.zip -4dbd01ab2582a74fab4d7ee8f9b11966d4188fba9eedcd31fcc0a5588cd0c702 snapraid-15.1-x86_64-1_am.txz -0740169f66c506c29846a5aef57d2b11ffe5c354c5bb0ba260f2b5f328f985c5 snapraid-15.2.tar.gz -208a5d99c94889e2357f0211d716bababcfb8289907167220563f734fb936e96 snapraid-15.2-1-x86_64.pkg.tar.zst -f1c37fa43beea8445858520a5cd514198d8c0c0ac5a5ab588adcd02e872ee681 snapraid-15.2-1.x86_64.rpm -882334e5102b418ace0753f9335a99783fc784e149429c29911247b56f04e438 snapraid-15.2-windows-x64.zip -942157645a532e2a280ca404625476c42591436f2292491b1faedb9d48efd07e snapraid-15.2-windows-x86.zip -7bb455cb77586dd2d90f6868c66dd9cdf6774aa89c099f1026a851407ac7adac snapraid-15.2-x86_64-1_am.txz 128b5b0f80a6a0911e77ed3b5d9d81529f5ea1d8fffbb5112e3af3467d80464a snapraid_14.0-1_amd64.deb b9c41ae2e8edbc413abd3172b7ed2da2cba7fa0e631d0f47fc1dafccf7c6e92c snapraid_14.1-1_amd64.deb e6456b926bb053d34645e31e5e3d8363e952c9869e873eb348abb1149598d331 snapraid_14.2-1_amd64.deb @@ -249,7 +225,3 @@ b71b556da4be6263a904fb8ac489933875467adbb920dd3911e54d87e9637ca3 snapraid_14.6- 35b156fc4d5c34339eed7e7d73e22a76e646879ecf6846b5ca20f7992ff55640 snapraid_14.7-1_amd64.deb c0e00a25b9bce40ff74f5a55401c3bfedac1881fd99aaa90a692f5858d417f7d snapraid_14.8-1_amd64.deb 09064689a64d965ac33ee89fb8b2cc14dfec0745f2b3b86a78f377fc353b7e07 snapraid_14.9-1_amd64.deb -15f5fd9f639912c94daae9d75d3567b7a3fc0e18475880f972acd8d8e9e0992c snapraid_14.10-1_amd64.deb -845091e7eeb2495fc3899dc6e8fab2c3fe7a5945f345c767a7603dab90e2cb84 snapraid_15.0-1_amd64.deb -974cd45fc8f44d5642b49bd9b30e81aca2bcd7689877f9fa73d1e58d3f1b5bda snapraid_15.1-1_amd64.deb -9c78f158347cc0340ed5c9877724bb4f3b5e5fb48b43332ee7e0e7b6a63642d9 snapraid_15.2-1_amd64.deb diff --git a/HISTORY b/HISTORY index 4e718e0e..18db39ff 100644 --- a/HISTORY +++ b/HISTORY @@ -1,36 +1,8 @@ SnapRAID HISTORY ================ -15.2 2026/11 -============ - * Avoid creating a redundant Windows Job Object when SnapRAID is - already running inside an existing job (such as under SnapRAID - Daemon). This prevents potential execution failures of child - processes (such as smartctl) on systems where nested job objects are - not supported. - * Skip disks on network or virtual filesystems (such as NFS, CIFS, - FUSE, or tmpfs) in device operations like 'up', 'down', 'smart', - and 'probe' on Linux, instead of failing with an error. If no - physical device remains, the operation completes successfully with - an informational message. - -15.1 2026/10 -============ - * Add synthetic device IDs on Linux, allowing SnapRAID Daemon to - preserve history for devices without unique hardware IDs while - they remain connected within the same boot. - * Fix a failed internal inconsistency check when '--gui-touch-before' - is used before the first sync when no content file exists yet. - Also ensure that commands that can save the content file ('touch', - 'scrub', 'rehash', 'rewrite') fail with a clear error message if the - array has not been synced yet. This fix is required by - SnapRAID-Daemon. - * Make test and doc variables in Makefile.am compatible with BSD make, - allowing builds, tests, and documentation generation on BSD systems - without requiring GNU make. - -15.0 2026/10 -============ +15.0 (Work in Progress) +======================= * Added support for automated filesystem snapshots to ensure data consistency and improve recovery reliability. When the 'snapshot' option is enabled, SnapRAID transparently manages @@ -68,7 +40,7 @@ SnapRAID HISTORY behaviour is preserved. Primarily intended for NixOS and other non-FHS systems. * Removed the 512 TiB limitation on hard disk size (when using the - default 256 KiB blocks). + default 128 KiB blocks). * Added support for a custom info options tag `[info: ...]` in the `smartctl` configuration option. This allows replacing the default `-a` command line argument used to retrieve SMART attributes with a diff --git a/LICENSES/BSD-2-Clause b/LICENSES/BSD-2-Clause index 6535ce92..6c86157a 100644 --- a/LICENSES/BSD-2-Clause +++ b/LICENSES/BSD-2-Clause @@ -1,3 +1,5 @@ +Copyright (c) 2010, Andrea Mazzoleni. All rights reserved. + Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/Makefile.am b/Makefile.am index 1da5e4b6..471f18b5 100644 --- a/Makefile.am +++ b/Makefile.am @@ -217,10 +217,7 @@ $(CLI_PREFIX)sync.$(OBJEXT): $(CLI_INCLUDES) $(CLI_PREFIX)thermal.$(OBJEXT): $(CLI_INCLUDES) $(CLI_PREFIX)touch.$(OBJEXT): $(CLI_INCLUDES) $(CLI_PREFIX)unixapp.$(OBJEXT): $(CLI_INCLUDES) -$(CLI_PREFIX)util.$(OBJEXT): $(CLI_INCLUDES) \ - $(srcdir)/cmdline/murmur3.c \ - $(srcdir)/cmdline/spooky2.c \ - $(srcdir)/cmdline/museair.c +$(CLI_PREFIX)util.$(OBJEXT): $(CLI_INCLUDES) # Add the .version file in the distribution dist-hook: @@ -228,28 +225,13 @@ dist-hook: LANG_CODES = de es fr it ja ko pl pt ro ru sv uk zh -LIST_LANG_D = \ - doc/snapraid-de.d \ - doc/snapraid-es.d \ - doc/snapraid-fr.d \ - doc/snapraid-it.d \ - doc/snapraid-ja.d \ - doc/snapraid-ko.d \ - doc/snapraid-pl.d \ - doc/snapraid-pt.d \ - doc/snapraid-ro.d \ - doc/snapraid-ru.d \ - doc/snapraid-sv.d \ - doc/snapraid-uk.d \ - doc/snapraid-zh.d +LIST_D = doc/snapraid.d doc/snapraid_log.d $(patsubst %, doc/snapraid-%.d, $(LANG_CODES)) -LIST_D = doc/snapraid.d doc/snapraid_log.d $(LIST_LANG_D) +LIST_MAN = doc/snapraid.1 doc/snapraid_log.1 $(patsubst %, doc/snapraid-%.1, $(LANG_CODES)) -LIST_MAN = $(LIST_D:.d=.1) +LIST_TXT = doc/snapraid.txt doc/snapraid_log.txt $(patsubst %, doc/snapraid-%.txt, $(LANG_CODES)) -LIST_TXT = $(LIST_D:.d=.txt) - -LIST_XML = $(LIST_D:.d=.xml) +LIST_XML = doc/snapraid.xml doc/snapraid_log.xml $(patsubst %, doc/snapraid-%.xml, $(LANG_CODES)) EXTRA_DIST = \ autogen.sh autover.sh valgrind.supp \ @@ -400,13 +382,8 @@ else # HAVE_POSIX CHECKCOUNT_DEFAULT = 500 endif # HAVE_POSIX -CHECKCOUNT_0 = $(CHECKCOUNT_DEFAULT) -CHECKCOUNT_1 = 500 -CHECKCOUNT = $(CHECKCOUNT_$(SHORTCHECK)) - -CHECKSIZE_0 = 4000 -CHECKSIZE_1 = 100 -CHECKSIZE = $(CHECKSIZE_$(SHORTCHECK)) +CHECKCOUNT = $(if $(filter 1,$(SHORTCHECK)),500,$(CHECKCOUNT_DEFAULT)) +CHECKSIZE = $(if $(filter 1,$(SHORTCHECK)),100,4000) CONF = $(srcdir)/test/test-par6.conf THERMAL = $(srcdir)/test/test-par6-thermal.conf diff --git a/TODO b/TODO index 614d56ef..2c1b77a4 100644 --- a/TODO +++ b/TODO @@ -6,23 +6,19 @@ This is the list of TODO items for SnapRAID. 16.x Releases ============= -* Support a `[power_control: off]` tag in `smartctl` options to disable power - management (`up` and `down`) on specific disks (e.g. for SAS drives that - fail to wake up on write). Emit an attribute tag in probe for snapraid-daemon. - -* Enable MuseAir and disable random seed +* Use statx() instead of lstat() when scanning directories. * Add a new 'restore' command, similar to 'fix', that recovers a whole disk using the same multithreaded approach as sync/scrub. -* Use statx() instead of lstat() when scanning directories. - * Integration with MergerFS ? * Restore ownership and permissions, at least in Unix. * Restore directory timestamps. +* Enable MuseAir and disable random seed + * Check Reliability Test. We can derive Failure probability from MBTF https://github.com/linuxhw/SMART https://github.com/linuxhw/EnterpriseDrive @@ -44,13 +40,6 @@ Future - Major -* Optimize `sync` on Btrfs by using `btrfs send -p --no-data` - to identify modified files and extent offsets, avoiding a full filesystem - metadata scan. Parity recomputation still reads from the latest snapshot. - Store the Btrfs snapshot generation (and UUID) in the content file to verify - that the parent snapshot strictly matches the content state before diffing, - falling back to a full scan if mismatched. - * Size-Based Checkpointing (autosave) using a write-ahead log (WAL). Instead of writing the full content file periodically (autosave), write incremental updates to companion `.wal` files. Use the existing bit-efficient `STREAM` diff --git a/cmdline/app.h b/cmdline/app.h index 69a2cdf4..f53a2d74 100644 --- a/cmdline/app.h +++ b/cmdline/app.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2026 Andrea Mazzoleni -#ifndef APP_H -#define APP_H +#ifndef __APP_H +#define __APP_H #include "os/os.h" @@ -87,7 +87,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har #define SNAPSHOT_SCAN "scan" #define SNAPSHOT_RETIRING_SUFFIX ".retiring" -/** +/* * Snapshots context */ struct fssnapshot_struct { @@ -213,7 +213,7 @@ int fssnapshot_replace(struct fssnapshot_struct* fss, const char* old_name, cons */ int fssnapshot_path(const struct fssnapshot_struct* fss, const char* name, char* path, size_t path_size); -/** +/* * Log file. * * This stream if fully buffered. @@ -257,7 +257,7 @@ extern int exit_degraded; */ #define SMART_FLAGS 256 -/** +/* * Counts command, transport, or controller-level errors reported by the device. * These reflect failed I/O operations not directly caused by media defects * (for example interface, firmware, or power-related errors). @@ -266,7 +266,7 @@ extern int exit_degraded; */ #define SMART_ERROR_PROTOCOL 257 -/** +/* * Counts media-level errors where data could not be reliably read or written. * These indicate actual storage surface or flash failures and may imply data loss. * This counter is cumulative and never resets to zero, even if the underlying @@ -387,7 +387,7 @@ struct devinfo_struct { char name[PATH_MAX]; /**< Name of the disk combined with the split index if any */ char mount[PATH_MAX]; /**< Mount point or other contained directory. */ char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Attributes to ignore */ char file[PATH_MAX]; /**< File device. */ #ifdef _WIN32 diff --git a/cmdline/bw.h b/cmdline/bw.h index 482202a5..9ba69e12 100644 --- a/cmdline/bw.h +++ b/cmdline/bw.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2016 Andrea Mazzoleni -#ifndef BW_H -#define BW_H +#ifndef __BW_H +#define __BW_H #include "state.h" #include "support.h" diff --git a/cmdline/check.c b/cmdline/check.c index 5dcbed5f..1a074244 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -14,7 +14,7 @@ #include "raid/raid.h" #include "raid/combo.h" -/** +/* * SnapRAID check/fix model * * Check verifies the data described by the content file against the files @@ -841,9 +841,11 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns * distinction disappears. */ + /************************************************************************/ + /* First strategy: parity represents CURRENT data. */ + /************************************************************************/ + /* - * First strategy: parity represents CURRENT data. - * * Under the CURRENT hypothesis: * * BLK hash CURRENT, parity normally synchronized @@ -1127,9 +1129,11 @@ static int repair(struct snapraid_state* state, int rehash, block_off_t pos, uns else log_tag("recover_sync:%" PRIu64 ":%u: Failed with %d attempts\n", pos, n, ret); + /************************************************************************/ + /* Second strategy: parity still represents OLD data. */ + /************************************************************************/ + /* - * Second strategy: parity still represents OLD data. - * * The CURRENT attempt failed, so retry under the opposite history * hypothesis: physical parity still represents the state before the * interrupted sync. @@ -1960,7 +1964,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna */ valid_parity = 1; - /* if the parity is used by at least one file */ + /* If the parity is used by at least one file */ used_parity = 0; /* keep track of the number of failed blocks */ diff --git a/cmdline/device.c b/cmdline/device.c index 2d494281..274e5c77 100644 --- a/cmdline/device.c +++ b/cmdline/device.c @@ -43,12 +43,12 @@ void device_name_set(devinfo_t* dev, const char* name, int index) */ #define SMART_MEASURES 256 -/** +/* * Divider for SMART attribute 5 */ static unsigned SMART_5_STEP = 1; -/** +/* * Failure rate for 30 days, for a disk * with SMART attribute 5 at a given value. */ @@ -107,12 +107,12 @@ static double SMART_5_R[SMART_MEASURES] = { 0.4249, }; -/** +/* * Divider for SMART attribute 187 */ static unsigned SMART_187_STEP = 1; -/** +/* * Failure rate for 30 days, for a disk * with SMART attribute 187 at a given value. */ @@ -171,12 +171,12 @@ static double SMART_187_R[SMART_MEASURES] = { 3.4520, }; -/** +/* * Divider for SMART attribute 188 */ static unsigned SMART_188_STEP = 1; -/** +/* * Failure rate for 30 days, for a disk * with SMART attribute 188 at a given value. */ @@ -235,12 +235,12 @@ static double SMART_188_R[SMART_MEASURES] = { 1.3731, }; -/** +/* * Divider for SMART attribute 197 */ static unsigned SMART_197_STEP = 1; -/** +/* * Failure rate for 30 days, for a disk * with SMART attribute 197 at a given value. */ @@ -299,12 +299,12 @@ static double SMART_197_R[SMART_MEASURES] = { 4.1131, }; -/** +/* * Divider for SMART attribute 198 */ static unsigned SMART_198_STEP = 1; -/** +/* * Failure rate for 30 days, for a disk * with SMART attribute 198 at a given value. */ @@ -423,7 +423,7 @@ static double smart_afr(devinfo_t* devinfo) } if ( - /* + /** * Don't check Command_Timeout (188) for Seagate disks. * * It's reported by users that for Archive SMR (Shingled Magnetic Recording) @@ -1237,16 +1237,13 @@ int state_device_force(struct snapraid_state* state, int operation, tommy_list* } } + /* distinguish skipped devices from unsupported device queries */ if (ret == 0 && tommy_list_empty(&low)) { + ret = -1; if (tommy_list_empty(&high)) { - ret = -1; log_error(ESOFT, "No accessible devices selected for this operation.\n"); goto bail; } - - /* not an error, the selected disks are on network or virtual filesystems without a block device to manage */ - msg_info("No physical devices found for the selected disks, nothing to do.\n"); - goto bail; } if (ret != 0) { diff --git a/cmdline/dup.c b/cmdline/dup.c index c1653e40..64e6352c 100644 --- a/cmdline/dup.c +++ b/cmdline/dup.c @@ -70,13 +70,9 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d return hash; } -static inline tommy_hash_t hash_hash(struct snapraid_hash* hash) +static inline tommy_uint32_t hash_hash(struct snapraid_hash* hash) { -#if TOMMY_SIZE_BIT == 64 - return tommy_hash_u64(0, hash->hash, HASH_MAX); -#else return tommy_hash_u32(0, hash->hash, HASH_MAX); -#endif } static void hash_free(void* void_hash) @@ -116,7 +112,7 @@ void state_dup(struct snapraid_state* state) for (j = disk->filelist; j != 0; j = j->next) { struct snapraid_file* file = j->data; struct snapraid_hash* hash; - tommy_hash_t hash_val; + tommy_hash_t hash32; /* if empty, skip it */ if (file->size == 0) @@ -128,9 +124,9 @@ void state_dup(struct snapraid_state* state) if (!hash) continue; - hash_val = hash_hash(hash); + hash32 = hash_hash(hash); - struct snapraid_hash* found = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash_val); + struct snapraid_hash* found = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash32); if (found) { ++count; size += found->file->size; @@ -138,7 +134,7 @@ void state_dup(struct snapraid_state* state) printf("%12" PRIu64 " %s = %s\n", file->size, fmt_term(disk, file->sub), fmt_term(found->disk, found->file->sub)); hash_free(hash); } else { - tommy_hashdyn_insert(&hashset, &hash->node, hash, hash_val); + tommy_hashdyn_insert(&hashset, &hash->node, hash, hash32); } } } diff --git a/cmdline/elem.c b/cmdline/elem.c index c349a002..60c63a9e 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -291,7 +291,7 @@ static int filter_element(tommy_list* filterlist, struct snapraid_filter** reaso { tommy_node* i; - /* + /** * The rule is that the latest filter defines the behaviour of paths * that doesn't match any filter. * @@ -375,7 +375,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub) if (lstat(path, &st) != 0) { /* if the file doesn't exist, we don't filter it out */ - if (errno == ENOENT || errno == ENOTDIR) + if (errno == ENOENT) return 0; /* LCOV_EXCL_START */ log_fatal(errno, "Error in stat file '%s'. %s.\n", path, strerror(errno)); @@ -470,7 +470,7 @@ int filter_snapshot(int enable, const char* sub, const char* name) return 0; } -struct snapraid_file* file_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) +struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) { struct snapraid_file* file; block_off_t blockmax; @@ -488,10 +488,8 @@ struct snapraid_file* file_alloc_len(unsigned block_size, const char* sub, size_ exit(EXIT_FAILURE); } - file = malloc_nofail(sizeof(struct snapraid_file) + sub_len + 1); - file->sub = file->sub_inline; - memcpy(file->sub, sub, sub_len); - file->sub[sub_len] = 0; + file = malloc_nofail(sizeof(struct snapraid_file)); + file->sub = strdup_nofail(sub); file->size = size; file->blockmax = blockmax; file->mtime_sec = mtime_sec; @@ -507,7 +505,6 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) { struct snapraid_file* file; block_off_t i; - size_t sub_len; if (copy->blockmax > BLOCK_MAX) { #if SIZE_MAX == UINT32_MAX @@ -518,11 +515,8 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) exit(EXIT_FAILURE); } - sub_len = strlen(copy->sub); - file = malloc_nofail(sizeof(struct snapraid_file) + sub_len + 1); - file->sub = file->sub_inline; - memcpy(file->sub, copy->sub, sub_len); - file->sub[sub_len] = 0; + file = malloc_nofail(sizeof(struct snapraid_file)); + file->sub = strdup_nofail(copy->sub); file->size = copy->size; file->blockmax = copy->blockmax; file->mtime_sec = copy->mtime_sec; @@ -545,8 +539,7 @@ void file_free(void* void_file) { struct snapraid_file* file = void_file; - if (file->sub != file->sub_inline) - free(file->sub); + free(file->sub); file->sub = 0; free(file->blockvec); file->blockvec = 0; @@ -555,9 +548,7 @@ void file_free(void* void_file) void file_rename(struct snapraid_file* file, const char* sub) { - if (file->sub != file->sub_inline) - free(file->sub); - + free(file->sub); file->sub = strdup_nofail(sub); } @@ -767,7 +758,6 @@ void extent_free(void* void_extent) free(extent); } -__aligned(32) int extent_parity_compare(const void* void_a, const void* void_b) { const struct snapraid_extent* arg_a = void_a; @@ -781,7 +771,6 @@ int extent_parity_compare(const void* void_a, const void* void_b) return 0; } -__aligned(32) int extent_file_compare(const void* void_a, const void* void_b) { const struct snapraid_extent* arg_a = void_a; @@ -803,13 +792,13 @@ int extent_file_compare(const void* void_a, const void* void_b) return 0; } -struct snapraid_link* link_alloc_len(const char* sub, size_t sub_len, const char* linkto, size_t linkto_len, unsigned link_flag) +struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag) { struct snapraid_link* slink; slink = malloc_nofail(sizeof(struct snapraid_link)); - slink->sub = strndup_nofail(sub, sub_len); - slink->linkto = strndup_nofail(linkto, linkto_len); + slink->sub = strdup_nofail(sub); + slink->linkto = strdup_nofail(linkto); slink->flag = link_flag; return slink; @@ -840,12 +829,12 @@ int link_alpha_compare(const void* void_a, const void* void_b) return strcmp(slink_a->sub, slink_b->sub); } -struct snapraid_dir* dir_alloc_len(const char* sub, size_t sub_len) +struct snapraid_dir* dir_alloc(const char* sub) { struct snapraid_dir* dir; dir = malloc_nofail(sizeof(struct snapraid_dir)); - dir->sub = strndup_nofail(sub, sub_len); + dir->sub = strdup_nofail(sub); dir->flag = 0; return dir; @@ -867,7 +856,7 @@ int dir_name_compare(const void* void_arg, const void* void_data) return strcmp(arg, dir->sub); } -struct snapraid_dealloc* dealloc_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec) +struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec) { struct snapraid_dealloc* dealloc; block_off_t blockmax; @@ -886,7 +875,7 @@ struct snapraid_dealloc* dealloc_alloc_len(unsigned block_size, const char* sub, } dealloc = malloc_nofail(sizeof(struct snapraid_dealloc) + blockmax * sizeof(struct snapraid_dealloc_block)); - dealloc->sub = strndup_nofail(sub, sub_len); + dealloc->sub = strdup_nofail(sub); dealloc->size = size; dealloc->mtime_sec = mtime_sec; dealloc->mtime_nsec = mtime_nsec; @@ -915,7 +904,7 @@ void dealloc_import(tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struc if (parity_pos == POS_NULL) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: Missing parity position for dealloc block %" PRIu64 "\n", i); - os_abort(); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -931,7 +920,7 @@ void dealloc_import(tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struc default : /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: State for dealloc block %" PRIu64 " state %u\n", i, block_state); - os_abort(); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } } @@ -1389,20 +1378,6 @@ static int extent_parity_inside_compare_unlock(const void* void_a, const void* v return 0; } -struct snapraid_extent* fs_par2extent_find_next(struct snapraid_disk* disk, block_off_t parity_pos) -{ - struct extent_parity_inside arg = { parity_pos }; - struct snapraid_extent* extent; - - fs_lock(disk); - - /* non-overlapping extents let this search find the containing extent or the next one */ - extent = tommy_tree_search_greater_equal_compare(&disk->fs_parity, extent_parity_inside_compare_unlock, &arg); - - fs_unlock(disk); - return extent; -} - /** * Search the extent at the specified parity position. * The search is optimized for sequential accesses. @@ -1604,8 +1579,8 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap extent = extent_alloc(parity_pos, file, file_pos, count); /* insert the extent in the trees */ - parity_extent = tommy_tree_insert_unique(&disk->fs_parity, &extent->parity_node, extent); - file_extent = tommy_tree_insert_unique(&disk->fs_file, &extent->file_node, extent); + parity_extent = tommy_tree_insert(&disk->fs_parity, &extent->parity_node, extent); + file_extent = tommy_tree_insert(&disk->fs_file, &extent->file_node, extent); if (parity_extent != extent || file_extent != extent) { /* LCOV_EXCL_START */ @@ -1689,8 +1664,8 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) second_extent = extent_alloc(extent->parity_pos + first_count + 1, extent->file, extent->file_pos + first_count + 1, second_count); /* insert the extent in the trees */ - parity_extent = tommy_tree_insert_unique(&disk->fs_parity, &second_extent->parity_node, second_extent); - file_extent = tommy_tree_insert_unique(&disk->fs_file, &second_extent->file_node, second_extent); + parity_extent = tommy_tree_insert(&disk->fs_parity, &second_extent->parity_node, second_extent); + file_extent = tommy_tree_insert(&disk->fs_file, &second_extent->file_node, second_extent); if (parity_extent != second_extent || file_extent != second_extent) { /* LCOV_EXCL_START */ @@ -1773,7 +1748,7 @@ void bucket_free(void* void_bucket) void bucket_insert(tommy_hashdyn* bucket_hash, time_t time_at, block_off_t count, int justsynced) { - tommy_hash_t hash = bucket_time_hash(time_at); + tommy_hash_t hash = tommy_inthash_u64(time_at); tommy_node* i = tommy_hashdyn_bucket(bucket_hash, hash); while (i) { struct snapraid_bucket* entry = i->data; diff --git a/cmdline/elem.h b/cmdline/elem.h index 7e1ef4a0..faba91b2 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef ELEM_H -#define ELEM_H +#ifndef __ELEM_H +#define __ELEM_H #include "util.h" #include "support.h" @@ -16,7 +16,7 @@ /****************************************************************************/ /* snapraid */ -/** +/* * Maximum number of block positions in a data/parity disk span. * * With the default 256 KiB block size, this limits the maximum supported @@ -341,8 +341,6 @@ struct snapraid_file { tommy_hashdyn_node nodeset; tommy_hashdyn_node pathset; tommy_hashdyn_node stampset; - - char sub_inline[]; /**< Inline storage for the initial sub path. */ }; /** @@ -415,7 +413,7 @@ struct snapraid_extra { char name[PATH_MAX]; /**< Name of the disk. */ char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */ char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */ char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */ char fstype[FSINFO_MAX]; /**< Filesystem type */ @@ -454,7 +452,7 @@ struct snapraid_disk { struct fssnapshot_struct fss; char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */ char uuid[UUID_MAX]; /**< UUID of the disk. They are probed during the config reading. */ char fstype[FSINFO_MAX]; /**< Filesystem type */ @@ -597,7 +595,7 @@ struct snapraid_parity { struct snapraid_split split_map[SPLIT_MAX]; /**< Parity splits. */ unsigned split_mac; /**< Number of parity splits. */ char smartctl[SMARTCTL_MAX]; /**< Custom option for smartctl. Empty means auto. */ - char smartctl_info[SMARTCTL_MAX]; /**< Info options for smartctl. Empty means -a. */ + char smartctl_info[SMARTCTL_MAX]; /* Info options for smartctl. Empty means -a. */ struct smartignore_struct smartignore[SMART_IGNORE_MAX]; /**< Smart attributes to ignore for this device. */ block_off_t total_blocks; /**< Number of total blocks. */ block_off_t free_blocks; /**< Number of free blocks at the last sync. */ @@ -910,11 +908,7 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask) /** * Allocate a file without initializing its blocks. */ -struct snapraid_file* file_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode); -static inline struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) -{ - return file_alloc_len(block_size, sub, strlen(sub), size, mtime_sec, mtime_nsec, inode); -} +struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode); /** * Duplicate a file. @@ -982,14 +976,9 @@ int file_path_compare(const void* void_a, const void* void_b); /** * Compute the hash of a file inode. */ -static inline tommy_hash_t file_inode_hash(uint64_t inode) +static inline tommy_uint32_t file_inode_hash(uint64_t inode) { -#if TOMMY_SIZE_BIT == 64 - return tommy_mulhash_u64(inode); -#else - /* avoid expensive 64-bit multiplications on 32-bit architectures */ - return tommy_inthash_u64(inode); -#endif + return (tommy_uint32_t)tommy_inthash_u64(inode); } /** @@ -1005,46 +994,20 @@ int file_stamp_compare(const void* void_a, const void* void_b); int file_namestamp_compare(const void* void_a, const void* void_b); int file_pathstamp_compare(const void* void_a, const void* void_b); -/** - * Compute the hash of a file path with known length. - */ -static inline tommy_hash_t file_path_hash_len(const char* sub, size_t len) -{ -#if TOMMY_SIZE_BIT == 64 - return tommy_hash_u64(0, sub, len); -#else - return tommy_hash_u32(0, sub, len); -#endif -} - /** * Compute the hash of a file path. */ -static inline tommy_hash_t file_path_hash(const char* sub) +static inline tommy_uint32_t file_path_hash(const char* sub) { - return file_path_hash_len(sub, strlen(sub)); + return tommy_hash_u32(0, sub, strlen(sub)); } /** * Compute the hash of a file stamp. */ -static inline tommy_hash_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec) +static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, int mtime_nsec) { - tommy_uint32_t nsec = mtime_nsec; - -#if TOMMY_SIZE_BIT == 64 - tommy_uint64_t size_key = size; - tommy_uint64_t sec_key = mtime_sec; - tommy_uint64_t nsec_key = nsec; - - /* valid nanoseconds use 30 bits above contemporary seconds; -1 stays distinct */ - return tommy_mulhash_u64(size_key ^ tommy_mulhash_u64(sec_key ^ (nsec_key << 34))); -#else - tommy_uint32_t size_key = size; - tommy_uint32_t sec_key = mtime_sec; - - return tommy_mulhash_u32(size_key ^ tommy_mulhash_u32(sec_key) ^ tommy_mulhash_u32(nsec)); -#endif + return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32((tommy_uint32_t)mtime_sec ^ tommy_inthash_u32(mtime_nsec))); } /** @@ -1096,11 +1059,7 @@ static inline unsigned link_flag_get(struct snapraid_link* slink, unsigned mask) /** * Allocate a link. */ -struct snapraid_link* link_alloc_len(const char* name, size_t name_len, const char* slink, size_t slink_len, unsigned link_flag); -static inline struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag) -{ - return link_alloc_len(name, strlen(name), slink, strlen(slink), link_flag); -} +struct snapraid_link* link_alloc(const char* name, const char* slink, unsigned link_flag); /** * Deallocate a link. @@ -1117,24 +1076,12 @@ int link_name_compare_to_arg(const void* void_arg, const void* void_data); */ int link_alpha_compare(const void* void_a, const void* void_b); -/** - * Compute the hash of a link name with known length. - */ -static inline tommy_hash_t link_name_hash_len(const char* name, size_t len) -{ -#if TOMMY_SIZE_BIT == 64 - return tommy_hash_u64(0, name, len); -#else - return tommy_hash_u32(0, name, len); -#endif -} - /** * Compute the hash of a link name. */ -static inline tommy_hash_t link_name_hash(const char* name) +static inline tommy_uint32_t link_name_hash(const char* name) { - return link_name_hash_len(name, strlen(name)); + return tommy_hash_u32(0, name, strlen(name)); } static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask) @@ -1155,11 +1102,7 @@ static inline void dir_flag_clear(struct snapraid_dir* dir, unsigned mask) /** * Allocate a dir. */ -struct snapraid_dir* dir_alloc_len(const char* name, size_t name_len); -static inline struct snapraid_dir* dir_alloc(const char* name) -{ - return dir_alloc_len(name, strlen(name)); -} +struct snapraid_dir* dir_alloc(const char* name); /** * Deallocate a dir. @@ -1171,34 +1114,18 @@ void dir_free(void* void_dir); */ int dir_name_compare(const void* void_arg, const void* void_data); -/** - * Compute the hash of a dir name with known length. - */ -static inline tommy_hash_t dir_name_hash_len(const char* name, size_t len) -{ -#if TOMMY_SIZE_BIT == 64 - return tommy_hash_u64(0, name, len); -#else - return tommy_hash_u32(0, name, len); -#endif -} - /** * Compute the hash of a dir name. */ -static inline tommy_hash_t dir_name_hash(const char* name) +static inline tommy_uint32_t dir_name_hash(const char* name) { - return dir_name_hash_len(name, strlen(name)); + return tommy_hash_u32(0, name, strlen(name)); } /** * Allocate a dealloc. */ -struct snapraid_dealloc* dealloc_alloc_len(unsigned block_size, const char* sub, size_t sub_len, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec); -static inline struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec) -{ - return dealloc_alloc_len(block_size, sub, strlen(sub), size, mtime_sec, mtime_nsec); -} +struct snapraid_dealloc* dealloc_alloc(unsigned block_size, const char* sub, data_off_t size, int64_t mtime_sec, int32_t mtime_nsec); /** * Deallocate a dealloc. @@ -1274,13 +1201,6 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t pos); */ struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos); -/** - * Find the first extent ending after the specified parity position. - * Return 0 if no such extent exists. The caller must ensure that the extent - * trees remain unchanged while using the returned extent or its tree node. - */ -struct snapraid_extent* fs_par2extent_find_next(struct snapraid_disk* disk, block_off_t parity_pos); - /** * Get the file position from the parity position. * Return 0 if no file is using it. @@ -1490,7 +1410,6 @@ static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos) * Grow the array once and process each internal block directly, avoiding the * per-position grow and lookup performed by info_set(). */ -__aligned_loops(64) static inline void info_set_run(tommy_arrayblkof* array, block_off_t pos, block_off_t count, snapraid_info info) { tommy_arrayblkof_grow(array, pos + count); @@ -1571,19 +1490,6 @@ struct snapraid_bucket { void bucket_free(void* void_bucket); -/** - * Compute the hash of a bucket time. - */ -static inline tommy_hash_t bucket_time_hash(time_t time_at) -{ -#if TOMMY_SIZE_BIT == 64 - return tommy_mulhash_u64(time_at); -#else - /* avoid expensive 64-bit multiplications on 32-bit architectures */ - return tommy_inthash_u64(time_at); -#endif -} - /** * Insert an bucket record into the hashtable */ diff --git a/cmdline/handle.c b/cmdline/handle.c index 34660054..11522209 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -113,7 +113,7 @@ int handle_open_create(struct snapraid_handle* handle, struct snapraid_file* fil * If creation was not requested, a missing file is an expected condition, * not a fatal error. */ - if (create || (errno != ENOENT && errno != ENOTDIR)) { + if (create || errno != ENOENT) { /* LCOV_EXCL_START */ log_fatal(errno, "Error opening file '%s'. %s.\n", handle->path, strerror(errno)); /* LCOV_EXCL_STOP */ diff --git a/cmdline/handle.h b/cmdline/handle.h index 757c939d..f117f6b8 100644 --- a/cmdline/handle.h +++ b/cmdline/handle.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef HANDLE_H -#define HANDLE_H +#ifndef __HANDLE_H +#define __HANDLE_H #include "state.h" #include "support.h" diff --git a/cmdline/import.h b/cmdline/import.h index 4676063f..8aef9b0d 100644 --- a/cmdline/import.h +++ b/cmdline/import.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef IMPORT_H -#define IMPORT_H +#ifndef __IMPORT_H +#define __IMPORT_H #include "elem.h" #include "state.h" diff --git a/cmdline/io.h b/cmdline/io.h index 92f4861b..df535946 100644 --- a/cmdline/io.h +++ b/cmdline/io.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2016 Andrea Mazzoleni -#ifndef IO_H -#define IO_H +#ifndef __IO_H +#define __IO_H #include "state.h" #include "support.h" diff --git a/cmdline/locate.h b/cmdline/locate.h index dc3a6cf9..57162559 100644 --- a/cmdline/locate.h +++ b/cmdline/locate.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2026 Andrea Mazzoleni -#ifndef LOCATE_H -#define LOCATE_H +#ifndef __LOCATE_H +#define __LOCATE_H /****************************************************************************/ /* snapraid */ diff --git a/cmdline/main.c b/cmdline/main.c index 0acd58b1..6d38d0bf 100644 --- a/cmdline/main.c +++ b/cmdline/main.c @@ -63,7 +63,7 @@ static int needs_quote(const WCHAR* arg) cmd[pos++] = (c); \ } while (0) -/** +/* * Append one argument using the Microsoft C runtime command-line * quoting rules. Use this when building a normal CreateProcessW() * command line that the target process will parse back into argv[]. @@ -146,10 +146,10 @@ static char* argutf8(const WCHAR* arg) return utf8_arg; } -/* global variable to store child process handle */ +/* Global variable to store child process handle */ static void* volatile child_process = NULL; -/** +/* * Safe atomic access helpers for child_process. * * The console_handler runs concurrently on an OS-managed thread created by @@ -169,7 +169,7 @@ static void child_process_write(HANDLE handle) InterlockedExchangePointer(&child_process, handle); } -/* console control handler - forwards Ctrl+C, Ctrl+Break to child */ +/* Console control handler - forwards Ctrl+C, Ctrl+Break to child */ static BOOL WINAPI console_handler(DWORD ctrl_type) { /* if no child, default behavior */ diff --git a/cmdline/memory.c b/cmdline/memory.c index ae600ea6..3402287b 100644 --- a/cmdline/memory.c +++ b/cmdline/memory.c @@ -14,44 +14,27 @@ */ static size_t mcounter; -#if HAVE_THREAD -static int memory_is_single_thread; - -void memory_single_thread(int single_thread) -{ - memory_is_single_thread = single_thread; -} - -static inline void malloc_counter_inc(size_t inc) -{ - if (tommy_likely(memory_is_single_thread)) - mcounter += inc; - else - __atomic_fetch_add(&mcounter, inc, __ATOMIC_RELAXED); -} - size_t malloc_counter_get(void) { - if (tommy_likely(memory_is_single_thread)) - return mcounter; - return __atomic_load_n(&mcounter, __ATOMIC_RELAXED); -} -#else -void memory_single_thread(int single_thread) -{ - (void)single_thread; + size_t ret; + + lock_memory(); + + ret = mcounter; + + unlock_memory(); + + return ret; } -static inline void malloc_counter_inc(size_t inc) +static void malloc_counter_inc(size_t inc) { + lock_memory(); + mcounter += inc; -} -size_t malloc_counter_get(void) -{ - return mcounter; + unlock_memory(); } -#endif /* LCOV_EXCL_START */ static ssize_t malloc_print(int f, const char* str) @@ -114,28 +97,23 @@ void* malloc_nofail(size_t size) /* LCOV_EXCL_STOP */ } +#ifndef CHECKER /* Don't preinitialize when running for valgrind */ + /* + * Here we preinitialize the memory to ensure that the OS is really allocating it + * and not only reserving the addressable space. + * Otherwise we are risking that the OOM (Out Of Memory) killer in Linux will kill the process. + * Filling the memory doesn't ensure to disable OOM, but it increase a lot the chances to + * get a real error from malloc() instead than a process killed. + * Note that calloc() doesn't have the same effect. + */ + memset(ptr, 0xA5, size); +#endif + malloc_counter_inc(size); return ptr; } -void* realloc_nofail(void* ptr, size_t size) -{ - /* keep zero-size requests allocated, as in malloc_nofail() */ - void* new_ptr = realloc(ptr, size ? size : 1); - - if (!new_ptr) { - /* LCOV_EXCL_START */ - malloc_fail(size); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - malloc_counter_inc(size); - - return new_ptr; -} - void* nalloc_nofail(size_t count, size_t size) { if (size != 0 && count > SIZE_MAX / size) { @@ -145,6 +123,7 @@ void* nalloc_nofail(size_t count, size_t size) size *= count; + /* see the note in malloc_nofail() of why we don't use calloc() */ void* ptr = malloc(size ? size : 1); if (!ptr) { @@ -161,21 +140,9 @@ void* nalloc_nofail(size_t count, size_t size) void* calloc_nofail(size_t count, size_t size) { - if (size != 0 && count > SIZE_MAX / size) { - log_fatal(EINTERNAL, "Allocation size overflow\n"); - os_abort(); - } + void* ptr = nalloc_nofail(count, size); - void* ptr = calloc(count ? count : 1, size ? size : 1); - - if (!ptr) { - /* LCOV_EXCL_START */ - malloc_fail(count * size); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - malloc_counter_inc(count * size); + memset(ptr, 0, count * size); return ptr; } @@ -191,13 +158,3 @@ char* strdup_nofail(const char* str) return ptr; } -char* strndup_nofail(const char* str, size_t size) -{ - char* ptr = malloc_nofail(size + 1); - - memcpy(ptr, str, size); - ptr[size] = 0; - - return ptr; -} - diff --git a/cmdline/memory.h b/cmdline/memory.h index 88e6e278..d2ed0b9b 100644 --- a/cmdline/memory.h +++ b/cmdline/memory.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef MEMORY_H -#define MEMORY_H +#ifndef __MEMORY_H +#define __MEMORY_H /****************************************************************************/ /* memory */ @@ -12,23 +12,12 @@ */ size_t malloc_counter_get(void); -/** - * Enable or disable single-thread mode for memory counters. - */ -void memory_single_thread(int single_thread); - /** * Safe malloc. * If no memory is available, it aborts. */ void* malloc_nofail(size_t size); -/** - * Safe realloc. - * If no memory is available, it aborts. - */ -void* realloc_nofail(void* ptr, size_t size); - /** * Safe malloc with calloc calling arguments. * If no memory is available, it aborts. @@ -47,12 +36,6 @@ void* calloc_nofail(size_t count, size_t size); */ char* strdup_nofail(const char* str); -/** - * Safe strndup with known length. - * If no memory is available, it aborts. - */ -char* strndup_nofail(const char* str, size_t size); - /** * Helper for printing an error about a failed allocation. */ diff --git a/cmdline/mingwapp.c b/cmdline/mingwapp.c index 23c28d10..992eb409 100644 --- a/cmdline/mingwapp.c +++ b/cmdline/mingwapp.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifdef __MINGW32__ /* only for MingW */ +#ifdef __MINGW32__ /* Only for MingW */ #include "support.h" #include "raid/cpu.h" @@ -82,7 +82,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har if ((attr & FILE_ATTRIBUTE_DIRECTORY) == 0) { char* slash = strrchr(dir, '/'); - /* + /** * Cut the file name, but leave the last slash. * * This is done because a MSDN comment about using of UNC paths. @@ -135,7 +135,7 @@ int fsinfo(const char* path, int* has_persistent_inode, int* has_syncronized_har if (has_persistent_inode) { *has_persistent_inode = 0; - /* + /** * Reuse stored file IDs across scans only for filesystems known to * provide stable and unique IDs. An unknown or unavailable filesystem * must be treated as volatile: a false negative only loses inode-based @@ -250,7 +250,7 @@ static int base64_encode(const unsigned char* in, size_t in_len, char* out, size return 0; } -/** +/* * PowerShell helper * * Runs a PowerShell one-liner via os_spawn() and captures the first line @@ -2411,7 +2411,7 @@ void machineinfo(struct machineinfo_struct* info) info->memory_page_size = si.dwPageSize; info->cpu_cores_logical = GetActiveProcessorCount(ALL_PROCESSOR_GROUPS); - /* query processor topology and cache hierarchy via GetLogicalProcessorInformationEx */ + /* Query processor topology and cache hierarchy via GetLogicalProcessorInformationEx */ DWORD len = 0; GetLogicalProcessorInformationEx(RelationAll, 0, &len); if (GetLastError() == ERROR_INSUFFICIENT_BUFFER && len > 0) { diff --git a/cmdline/mkstream.c b/cmdline/mkstream.c index 146110fe..224e8380 100644 --- a/cmdline/mkstream.c +++ b/cmdline/mkstream.c @@ -144,7 +144,7 @@ void test(int advise_mode) char copy[STR_MAX]; memset(str, ' ' + j, j - 1); str[j - 1] = 0; - if (sgetbs(s, copy, sizeof(copy)) != (ssize_t)(j - 1) || strcmp(copy, str) != 0) { + if (sgetbs(s, copy, sizeof(copy)) != 0 || strcmp(copy, str) != 0) { /* LCOV_EXCL_START */ exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ diff --git a/cmdline/murmur3.c b/cmdline/murmur3.c index 8f912a7b..9ac614db 100644 --- a/cmdline/murmur3.c +++ b/cmdline/murmur3.c @@ -14,7 +14,7 @@ // MurmurHash3 was written by Austin Appleby, and is placed in the public // domain. The author hereby disclaims copyright to this source code. -/* finalization mix - force all bits of a hash block to avalanche */ +/* Finalization mix - force all bits of a hash block to avalanche */ static inline uint32_t fmix32(uint32_t h) { h ^= h >> 16; @@ -25,17 +25,43 @@ static inline uint32_t fmix32(uint32_t h) return h; } -/** - * Non-static variables are intentionally used instead of static const - * because keeping these constants in CPU registers yields measurably - * higher throughput than using 32-bit immediate constants in imul. +/* + * Warning! + * Don't declare these variables static, otherwise the gcc optimizer + * may generate very slow code for multiplication with these constants, + * like: + + -> .cpp + k1 *= c1; + -> .asm + 152: 8d 14 80 lea (%eax,%eax,4),%edx + 155: 8d 14 90 lea (%eax,%edx,4),%edx + 158: c1 e2 03 shl $0x3,%edx + 15b: 29 c2 sub %eax,%edx + 15d: 8d 14 d2 lea (%edx,%edx,8),%edx + 160: 8d 14 90 lea (%eax,%edx,4),%edx + 163: 8d 14 d0 lea (%eax,%edx,8),%edx + 166: 8d 14 90 lea (%eax,%edx,4),%edx + 169: 8d 14 50 lea (%eax,%edx,2),%edx + 16c: 8d 14 90 lea (%eax,%edx,4),%edx + 16f: 8d 14 92 lea (%edx,%edx,4),%edx + 172: 8d 14 50 lea (%eax,%edx,2),%edx + 175: 8d 04 d0 lea (%eax,%edx,8),%eax + 178: 8d 14 c5 00 00 00 00 lea 0x0(,%eax,8),%edx + 17f: 29 d0 sub %edx,%eax + + * resulting in speeds of 500 MB/s instead of 3000 MB/s. + * + * Verified with gcc 4.4.4 compiling with : + * + * g++ -g -c -O2 MurmurHash3.cpp -o MurmurHash3.o */ uint32_t c1 = 0x239b961b; uint32_t c2 = 0xab0e9789; uint32_t c3 = 0x38b34ae5; uint32_t c4 = 0xa1e38b93; -static void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, void* digest) +void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* seed, void* digest) { const uint8_t* p; const uint8_t* end; @@ -51,11 +77,6 @@ static void MurmurHash3_x86_128(const void* data, size_t size, const uint8_t* se end = p + (size & ~15); /* body */ -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC unroll 8 -#elif defined(__clang__) -#pragma unroll 8 -#endif while (p < end) { uint32_t k1 = util_read32(p + 0); uint32_t k2 = util_read32(p + 4); diff --git a/cmdline/museair.c b/cmdline/museair.c index 2f765e9e..e6db6046 100644 --- a/cmdline/museair.c +++ b/cmdline/museair.c @@ -20,7 +20,7 @@ * of this license, visit: https://creativecommons.org/publicdomain/zero/1.0/ */ -/** +/* * Analysis of the BFast variant and blinding multiplication * * Many fast non-cryptographic hashes (wyhash, rapidhash) use "folding @@ -150,7 +150,7 @@ static const uint64_t MUSEAIR_CONSTANT[13] = { do { \ l = a; \ h = b; \ - tommy_mum(&l, &h); \ + util_mum(&l, &h); \ } while (0) #define likely(a) tommy_likely(a) @@ -190,11 +190,6 @@ static void MuseAirLoong(const void* bytes, size_t len, const uint8_t* seed, uin state[5] ^= seed_b & MASK_K; if (unlikely(q > u64x(12))) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC unroll 2 -#elif defined(__clang__) -#pragma unroll 2 -#endif do { state[0] ^= util_read64(p + u64x(0)); state[1] ^= util_read64(p + u64x(1)); diff --git a/cmdline/parity.c b/cmdline/parity.c index 64643679..5ea3b735 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -220,7 +220,7 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p /* LCOV_EXCL_STOP */ } - /* + /** * If the parity size is not yet set, set it now. * This happens when adding a new parity level or split that has * no persisted size yet. @@ -625,7 +625,7 @@ int parity_restore(struct snapraid_parity_handle* handle, data_off_t size, uint3 if ((run & block_mask) != 0) { /* LCOV_EXCL_START */ errno = ENXIO; - log_fatal(EINTERNAL, "Split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); + log_fatal(EINTERNAL, "Internal inconsistency in split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); return -1; /* LCOV_EXCL_STOP */ } @@ -731,7 +731,7 @@ int parity_chsize(struct snapraid_parity_handle* handle, struct snapraid_parity* if ((run & block_mask) != 0) { /* LCOV_EXCL_START */ errno = ENXIO; - log_fatal(EINTERNAL, "Split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); + log_fatal(EINTERNAL, "Internal inconsistency in split '%s' size with extra '%" PRIu64 "' bytes.\n", split->path, run & block_mask); return -1; /* LCOV_EXCL_STOP */ } @@ -884,7 +884,7 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par /* LCOV_EXCL_STOP */ } - /* + /** * If the parity size is not yet set, set it now. * This happens when adding a new parity level or split that has * no persisted size yet. diff --git a/cmdline/parity.h b/cmdline/parity.h index 91140492..dab95dd7 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef PARITY_H -#define PARITY_H +#ifndef __PARITY_H +#define __PARITY_H #include "support.h" #include "bw.h" diff --git a/cmdline/pool.c b/cmdline/pool.c index a386f623..5a46fd49 100644 --- a/cmdline/pool.c +++ b/cmdline/pool.c @@ -30,13 +30,9 @@ struct snapraid_pool* pool_alloc(const char* dir, const char* name, const char* return pool; } -static inline tommy_hash_t pool_hash(const char* file) +static inline tommy_uint32_t pool_hash(const char* file) { -#if TOMMY_SIZE_BIT == 64 - return tommy_hash_u64(0, file, strlen(file)); -#else return tommy_hash_u32(0, file, strlen(file)); -#endif } static void pool_free(void* void_pool) diff --git a/cmdline/scan.c b/cmdline/scan.c index 1664ca05..4eb0e4e3 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -20,8 +20,8 @@ static const char* es(int err) * If the file metadata (size and timestamp) is known to be synchronized with disk. * * Always set on POSIX where stat()/lstat() returns coherent metadata. On Windows NTFS, - * directory enumeration may return stale size/mtime for open/modified files and hardlinks - * until synchronized via lstat_sync() (GetFileInformationByHandle). + * directory enumeration may return stale size/mtime for hardlinks until synchronized + * via lstat_sync() (GetFileInformationByHandle). */ #define DISCOVERY_IS_SCAN_SYNCED 0x1 @@ -1021,7 +1021,7 @@ static void scan_file_apply(void* void_scan, void* void_disc) * in such case we have a "copy" over an "update" */ if (!state->opt.force_nocopy) { - tommy_hash_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec); + tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec); /* search for a file with the same name and stamp in all the disks */ for (i = state->disklist; i != 0; i = i->next) { @@ -1153,7 +1153,7 @@ static void scan_file_discover(struct snapraid_scan* scan, const char* sub, stru disc = discovery_alloc(sub, st); - /* hardlink collision detection */ + /* Hardlink collision detection */ if (disc->inode != INODE_INVALID) { struct snapraid_discovery* rep = tommy_hashdyn_search(&scan->file_discovery_inodeset, discovery_inode_compare_to_arg, &disc->inode, file_inode_hash(disc->inode)); if (!rep) { @@ -1161,7 +1161,7 @@ static void scan_file_discover(struct snapraid_scan* scan, const char* sub, stru disc->flag |= DISCOVERY_IS_CANONICAL; tommy_hashdyn_insert(&scan->file_discovery_inodeset, &disc->nodeset, disc, file_inode_hash(disc->inode)); } else { - /* inode collision detected! */ + /* Inode collision detected! */ #if HAVE_LSTAT_SYNC if (disk->has_volatile_hardlinks && !(rep->flag & DISCOVERY_IS_SCAN_SYNCED)) { char path_next[PATH_MAX]; @@ -1655,24 +1655,13 @@ static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* pa if (!st) st = DSTAT(path_next, dd, &st_buf); - existing = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub_next, file_path_hash(sub_next)); - if (existing) - file_flag_set(existing, FILE_IS_DISCOVERED); - #if HAVE_LSTAT_SYNC /* * In Windows fast directory enumeration, st_ino may be missing (0). - * In addition, directory enumeration may report stale size and mtime - * for files currently open or being written to. - * Refresh via lstat_sync() if st_ino is missing, or if the metadata - * is not synchronized and the file is new or modified. + * Only st_ino is required for file identity and hardlink tracking; + * we call lstat_sync() only when st_ino is missing to avoid expensive file opens. */ - if (st->st_ino == INODE_INVALID - || (st->st_sync == 0 - && (!existing - || existing->size != st->st_size - || existing->mtime_sec != st->st_mtime - || existing->mtime_nsec != STAT_NSEC(st)))) { + if (st->st_ino == INODE_INVALID) { if (lstat_sync(path_next, st, 0) != 0) { /* LCOV_EXCL_START */ log_tag("%s:%u:%s:%s: Stat error. %s.\n", es(errno), 0, disk->name, esc_tag(path_next), strerror(errno)); @@ -1683,6 +1672,10 @@ static int scan_sub(struct snapraid_scan* scan, int level, int is_diff, char* pa } #endif + existing = tommy_hashdyn_search(&disk->pathset, file_path_compare_to_arg, sub_next, file_path_hash(sub_next)); + if (existing) + file_flag_set(existing, FILE_IS_DISCOVERED); + scan_file_discover(scan, sub_next, st); processed = 1; } else { @@ -1944,8 +1937,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) * - Each disk is read only by its scan thread, enumerating files and populating * file_discovery_list without observable SnapRAID classification side effects. * - No old file, link, directory, or parity allocation is removed. - * - New/modified files and inode collisions on volatile filesystems trigger selective - * lstat_sync() to obtain authoritative metadata before Apply. + * - Inode collisions on volatile-hardlink filesystems trigger selective lstat_sync() + * to normalize authoritative metadata across hardlink groups before Apply. */ for (i = scanlist; i != 0; i = i->next) { struct snapraid_scan* scan = i->data; diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 2a2ae827..481e6d1d 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -11,7 +11,7 @@ #include "io.h" #include "raid/raid.h" -/** +/* * SnapRAID scrub model * * Scrub periodically verifies data and parity without modifying either of diff --git a/cmdline/search.c b/cmdline/search.c index 943802e8..bc4fdcdd 100644 --- a/cmdline/search.c +++ b/cmdline/search.c @@ -22,7 +22,7 @@ static const char* es(int err) static void search_file(struct snapraid_state* state, const char* path, data_off_t size, int64_t mtime_sec, int mtime_nsec) { struct snapraid_search_file* file; - tommy_hash_t file_hash; + tommy_uint32_t file_hash; file = malloc_nofail(sizeof(struct snapraid_search_file)); file->path = strdup_nofail(path); @@ -133,7 +133,7 @@ int search_file_compare(const void* void_arg, const void* void_data) int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer) { struct snapraid_search_file* file; - tommy_hash_t file_hash; + tommy_uint32_t file_hash; struct search_file_compare_arg arg; arg.state = state; diff --git a/cmdline/search.h b/cmdline/search.h index dc84747e..55c8fef3 100644 --- a/cmdline/search.h +++ b/cmdline/search.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2014 Andrea Mazzoleni -#ifndef SEARCH_H -#define SEARCH_H +#ifndef __SEARCH_H +#define __SEARCH_H #include "elem.h" #include "state.h" diff --git a/cmdline/selftest.c b/cmdline/selftest.c index cf6d5ead..a88f61e5 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -51,25 +51,25 @@ struct hash_test_vector { * Test vectors for tommy_hash32 */ static struct hash32_test_vector TEST_HASH32[] = { - { "", 0, 0xaffe48d7, 0xa766795d }, - { "a", 1, 0x2ae4a38d, 0xa766795d }, - { "abc", 3, 0x0aefd0ef, 0xa766795d }, - { "message digest", 14, 0x828c4a8a, 0xa766795d }, - { "abcdefghijklmnopqrstuvwxyz", 26, 0xc8072ea9, 0xa766795d }, - { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0xe8e24812, 0xa766795d }, - { "The quick brown fox jumps over the lazy dog", 43, 0xae34960f, 0xa766795d }, - { "\x00", 1, 0xac5373be, 0xa766795d }, - { "\x16\x27", 2, 0x321c4629, 0xa766795d }, - { "\xe2\x56\xb4", 3, 0x66b8fbd8, 0xa766795d }, - { "\xc9\x4d\x9c\xda", 4, 0xbc37eb40, 0xa766795d }, - { "\x79\xf1\x29\x69\x5d", 5, 0xb96d1628, 0xa766795d }, - { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x760cf131, 0xa766795d }, - { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0xd1d5a37a, 0xa766795d }, - { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x995d591f, 0xa766795d }, - { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0x8c7cfa48, 0xa766795d }, - { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0xaa8fa306, 0xa766795d }, - { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0x9c1d4af8, 0xa766795d }, - { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x1a2214ec, 0xa766795d }, + { "", 0, 0x8614384c, 0xa766795d }, + { "a", 1, 0x12c16c36, 0xa766795d }, + { "abc", 3, 0xc58e8af5, 0xa766795d }, + { "message digest", 14, 0x006b32f1, 0xa766795d }, + { "abcdefghijklmnopqrstuvwxyz", 26, 0x7e6fcfe0, 0xa766795d }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0x8604adf8, 0xa766795d }, + { "The quick brown fox jumps over the lazy dog", 43, 0xdeba3d3a, 0xa766795d }, + { "\x00", 1, 0x4a7d1c33, 0xa766795d }, + { "\x16\x27", 2, 0x8b50899b, 0xa766795d }, + { "\xe2\x56\xb4", 3, 0x60406493, 0xa766795d }, + { "\xc9\x4d\x9c\xda", 4, 0xa049144a, 0xa766795d }, + { "\x79\xf1\x29\x69\x5d", 5, 0x4da2c2f1, 0xa766795d }, + { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x59de30cf, 0xa766795d }, + { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x219e149c, 0xa766795d }, + { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x25067520, 0xa766795d }, + { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0xa1f368d8, 0xa766795d }, + { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0x805fc63d, 0xa766795d }, + { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0x7f75dd0f, 0xa766795d }, + { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0xb9154382, 0xa766795d }, { 0, 0, 0, 0 } }; @@ -77,29 +77,29 @@ static struct hash32_test_vector TEST_HASH32[] = { * Test vectors for tommy_strhash32 */ struct strhash32_test_vector TEST_STRHASH32[] = { - { "", 0xaffe48d7, 0xa766795d }, - { "a", 0x2ae4a38d, 0xa766795d }, - { "abc", 0x0aefd0ef, 0xa766795d }, - { "message digest", 0x828c4a8a, 0xa766795d }, - { "abcdefghijklmnopqrstuvwxyz", 0xc8072ea9, 0xa766795d }, - { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xe8e24812, 0xa766795d }, - { "The quick brown fox jumps over the lazy dog", 0xae34960f, 0xa766795d }, - { "\xff", 0x9ab96198, 0xa766795d }, - { "\x16\x27", 0x321c4629, 0xa766795d }, - { "\xe2\x56\xb4", 0x66b8fbd8, 0xa766795d }, - { "\xc9\x4d\x9c\xda", 0xbc37eb40, 0xa766795d }, - { "\x79\xf1\x29\x69\x5d", 0xb96d1628, 0xa766795d }, - { "\xff\x7e\xdf\x1e\x31\x1c", 0xaf63217b, 0xa766795d }, - { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 0xd1d5a37a, 0xa766795d }, - { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 0x995d591f, 0xa766795d }, - { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 0x8c7cfa48, 0xa766795d }, - { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 0xaa8fa306, 0xa766795d }, - { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\xff\xb7\xae", 0x59437055, 0xa766795d }, - { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 0x1a2214ec, 0xa766795d }, - { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 0xb5d1be68, 0xa766795d }, - { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 0xb2475cc8, 0xa766795d }, - { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 0x3cf85197, 0xa766795d }, - { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 0x8ec35c09, 0xa766795d }, + { "", 0x0af1416d, 0xa766795d }, + { "a", 0x68fa0f3f, 0xa766795d }, + { "abc", 0xfc68ffc5, 0xa766795d }, + { "message digest", 0x08477b63, 0xa766795d }, + { "abcdefghijklmnopqrstuvwxyz", 0x5b9c25e5, 0xa766795d }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x1e530ce7, 0xa766795d }, + { "The quick brown fox jumps over the lazy dog", 0xaf93eefe, 0xa766795d }, + { "\xff", 0xfc88801b, 0xa766795d }, + { "\x16\x27", 0xcd7216db, 0xa766795d }, + { "\xe2\x56\xb4", 0x05f98d02, 0xa766795d }, + { "\xc9\x4d\x9c\xda", 0xf65206f8, 0xa766795d }, + { "\x79\xf1\x29\x69\x5d", 0x72bd6bda, 0xa766795d }, + { "\xff\x7e\xdf\x1e\x31\x1c", 0x57dfb9b4, 0xa766795d }, + { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 0x499ff634, 0xa766795d }, + { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 0xe896b7ce, 0xa766795d }, + { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 0xfe3939f0, 0xa766795d }, + { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 0x4351d482, 0xa766795d }, + { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\xff\xb7\xae", 0x88e92135, 0xa766795d }, + { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 0x01109c16, 0xa766795d }, + { "\x87\xd8\x61\x61\x4c\x89\x17\x4e\xa1\xa4\xef\x13\xa9", 0xbcb050dc, 0xa766795d }, + { "\xfe\xa6\x5b\xc2\xda\xe8\x95\xd4\x64\xab\x4c\x39\x58\x29", 0xbe5e1fd5, 0xa766795d }, + { "\x94\x49\xc0\x78\xa0\x80\xda\xc7\x71\x4e\x17\x37\xa9\x7c\x40", 0x70d8c97f, 0xa766795d }, + { "\x53\x7e\x36\xb4\x2e\xc9\xb9\xcc\x18\x3e\x9a\x5f\xfc\xb7\xb0\x61", 0x957440a9, 0xa766795d }, { 0, 0, 0 } }; @@ -107,25 +107,25 @@ struct strhash32_test_vector TEST_STRHASH32[] = { * Test vectors for tommy_hash64 */ static struct hash64_test_vector TEST_HASH64[] = { - { "", 0, 0xf64d2d2164ca4e35ULL, 0x2f022773a766795dULL }, - { "a", 1, 0xcb0b7863fd912608ULL, 0x2f022773a766795dULL }, - { "abc", 3, 0x023adbd4412dfbc2ULL, 0x2f022773a766795dULL }, - { "message digest", 14, 0x8d16f620b58a030eULL, 0x2f022773a766795dULL }, - { "abcdefghijklmnopqrstuvwxyz", 26, 0xa615ea6b87f2b63fULL, 0x2f022773a766795dULL }, - { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0xb9f145676ebdb4fcULL, 0x2f022773a766795dULL }, - { "The quick brown fox jumps over the lazy dog", 43, 0x99b05c9548b523dcULL, 0x2f022773a766795dULL }, - { "\x00", 1, 0x432204e0ea8617fbULL, 0x2f022773a766795dULL }, - { "\x16\x27", 2, 0x5118c58166c6ab3eULL, 0x2f022773a766795dULL }, - { "\xe2\x56\xb4", 3, 0x29484604a9976acbULL, 0x2f022773a766795dULL }, - { "\xc9\x4d\x9c\xda", 4, 0xf000ac44a9f98f4aULL, 0x2f022773a766795dULL }, - { "\x79\xf1\x29\x69\x5d", 5, 0x9603fc74ec52e5aaULL, 0x2f022773a766795dULL }, - { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0xd39f549b363c8e2bULL, 0x2f022773a766795dULL }, - { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x1be47d9f53a2a31dULL, 0x2f022773a766795dULL }, - { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0xe3fa24e31af8c4ebULL, 0x2f022773a766795dULL }, - { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0xef8c3dcec426a25dULL, 0x2f022773a766795dULL }, - { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0xed0be01a88bde7cbULL, 0x2f022773a766795dULL }, - { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0xf9b50d9db5dfa543ULL, 0x2f022773a766795dULL }, - { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x695329dad2094e8aULL, 0x2f022773a766795dULL }, + { "", 0, 0x8614384cb5165fbfULL, 0x2f022773a766795dULL }, + { "a", 1, 0x1a2e0298a8e94a3dULL, 0x2f022773a766795dULL }, + { "abc", 3, 0x7555796b7a7d21ebULL, 0x2f022773a766795dULL }, + { "message digest", 14, 0x9411a57d04b92fb4ULL, 0x2f022773a766795dULL }, + { "abcdefghijklmnopqrstuvwxyz", 26, 0x3ca3f8d2b4e69832ULL, 0x2f022773a766795dULL }, + { "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 62, 0x6dae542ba0015a4dULL, 0x2f022773a766795dULL }, + { "The quick brown fox jumps over the lazy dog", 43, 0xe06d8cbb3d2ea1a6ULL, 0x2f022773a766795dULL }, + { "\x00", 1, 0x201e664fb5f2c021ULL, 0x2f022773a766795dULL }, + { "\x16\x27", 2, 0xef42fa8032c4b775ULL, 0x2f022773a766795dULL }, + { "\xe2\x56\xb4", 3, 0x6e6c498a6688466cULL, 0x2f022773a766795dULL }, + { "\xc9\x4d\x9c\xda", 4, 0x5195005419905423ULL, 0x2f022773a766795dULL }, + { "\x79\xf1\x29\x69\x5d", 5, 0x221235b48afee7c1ULL, 0x2f022773a766795dULL }, + { "\x00\x7e\xdf\x1e\x31\x1c", 6, 0x1b1f18b9266f095bULL, 0x2f022773a766795dULL }, + { "\x2a\x4c\xe1\xff\x9e\x6f\x53", 7, 0x2cbafa8e741d49caULL, 0x2f022773a766795dULL }, + { "\xba\x02\xab\x18\x30\xc5\x0e\x8a", 8, 0x4677f04c06e0758dULL, 0x2f022773a766795dULL }, + { "\xec\x4e\x7a\x72\x1e\x71\x2a\xc9\x33", 9, 0x5afe09e8214e2163ULL, 0x2f022773a766795dULL }, + { "\xfd\xe2\x9c\x0f\x72\xb7\x08\xea\xd0\x78", 10, 0x115b6276d209fab6ULL, 0x2f022773a766795dULL }, + { "\x65\xc4\x8a\xb8\x80\x86\x9a\x79\x00\xb7\xae", 11, 0xd0636d2f01cf3a3eULL, 0x2f022773a766795dULL }, + { "\x77\xe9\xd7\x80\x0e\x3f\x5c\x43\xc8\xc2\x46\x39", 12, 0x6d259f5fef74f93eULL, 0x2f022773a766795dULL }, { 0, 0, 0, 0 } }; @@ -188,13 +188,6 @@ static void test_hash(void) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - digest = tommy_hash_u32(TEST_STRHASH32[i].seed, buffer_aligned, strlen(TEST_STRHASH32[i].data)); - if (digest != TEST_STRHASH32[i].digest) { - /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "Failed strhash32 test\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } } for (i = 0; TEST_HASH64[i].data; ++i) { @@ -463,7 +456,7 @@ static void test_tommy(void) tommy_arrayblkof_init(&arrayblkof, sizeof(unsigned)); for (i = 0; i < TOMMY_SIZE; ++i) { - tommy_array_insert_tail(&array, &node[i]); + tommy_array_insert(&array, &node[i]); tommy_arrayblkof_grow(&arrayblkof, i + 1); *(unsigned*)tommy_arrayblkof_ref(&arrayblkof, i) = i; } @@ -591,11 +584,7 @@ static void test_tommy(void) tommy_tree_insert(&tree, &node[i], (void*)(uintptr_t)(i + 1)); /* try to insert a duplicate, count should not change */ - if (tommy_tree_insert_unique(&tree, &node[TOMMY_SIZE], (void*)(uintptr_t)1) != (void*)(uintptr_t)1) { - /* LCOV_EXCL_START */ - goto bail; - /* LCOV_EXCL_STOP */ - } + tommy_tree_insert(&tree, &node[TOMMY_SIZE], (void*)(uintptr_t)1); if (tommy_tree_count(&tree) != TOMMY_SIZE) { /* LCOV_EXCL_START */ @@ -1990,13 +1979,13 @@ static void test_smart_ignore(void) memset(&state, 0, sizeof(state)); memset(&devinfo, 0, sizeof(devinfo)); - /* initialize devinfo attributes with mock data */ + /* Initialize devinfo attributes with mock data */ for (i = 0; i < SMART_COUNT; ++i) { devinfo.smart[i].raw = 100; devinfo.smart[i].norm = 100; snprintf(devinfo.smart[i].name, sizeof(devinfo.smart[i].name), "Attr_%d", i); } - /* special name for test */ + /* Special name for test */ pathcpy(devinfo.smart[5].name, sizeof(devinfo.smart[5].name), "Reallocated_Sector_Ct"); pathcpy(devinfo.smart[197].name, sizeof(devinfo.smart[197].name), "Current_Pending_Sector"); @@ -2018,7 +2007,7 @@ static void test_smart_ignore(void) state_smart_ignore(&state, &devinfo); - /* check that ignored attributes are set to SMART_UNASSIGNED */ + /* Check that ignored attributes are set to SMART_UNASSIGNED */ if (devinfo.smart[5].raw != SMART_UNASSIGNED || devinfo.smart[5].norm != SMART_UNASSIGNED) { log_fatal(EINTERNAL, "test_smart_ignore: numerical global ignore failed\n"); exit(EXIT_FAILURE); @@ -2036,7 +2025,7 @@ static void test_smart_ignore(void) exit(EXIT_FAILURE); } - /* check that non-ignored attributes are still present */ + /* Check that non-ignored attributes are still present */ if (devinfo.smart[6].raw != 100) { log_fatal(EINTERNAL, "test_smart_ignore: unmodified attribute was cleared\n"); exit(EXIT_FAILURE); @@ -2194,30 +2183,6 @@ static void test_stream(void) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - - /* sgetbs test */ - unsigned char buf_bs[] = { 0x85, 'h', 'e', 'l', 'l', 'o' }; - char str_bs[16]; - memset(&f, 0, sizeof(f)); - f.pos = buf_bs; - f.end = buf_bs + sizeof(buf_bs); - if (sgetbs(&f, str_bs, sizeof(str_bs)) != 5 || strcmp(str_bs, "hello") != 0) { - /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "test_stream: sgetbs valid string failed\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* sgetbs buffer too small test */ - memset(&f, 0, sizeof(f)); - f.pos = buf_bs; - f.end = buf_bs + sizeof(buf_bs); - if (sgetbs(&f, str_bs, 5) >= 0) { - /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "test_stream: sgetbs small buffer accepted\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } } static void test_extent_run(void) @@ -2256,135 +2221,6 @@ static void test_extent_run(void) file_free(file); } -static void test_unsynced_sweep(void) -{ - struct snapraid_state state; - struct snapraid_disk* disk1; - struct snapraid_disk* disk2; - struct snapraid_file* file1; - struct snapraid_file* file2; - struct snapraid_file* file3; - struct snapraid_file* file4; - struct snapraid_file* del; - block_off_t i; - static const struct { - block_off_t blockmax; - uint64_t unsynced; - } bounds[] = { - { 0, 0 }, { 10, 0 }, { 15, 0 }, { 16, 1 }, { 19, 4 }, - { 20, 5 }, { 21, 5 }, { 22, 6 }, { 23, 7 }, { 29, 7 }, - { 30, 7 }, { 31, 8 }, { 32, 9 }, { 33, 10 }, { 34, 11 }, - { 100, 11 } - }; - - memset(&state, 0, sizeof(state)); - state.block_size = 4096; - tommy_list_init(&state.disklist); - - if (fs_count_unsynced(&state, 0, 100) != 0) { - log_fatal(EINTERNAL, "test_unsynced_sweep: no disks failed\n"); - exit(EXIT_FAILURE); - } - - disk1 = disk_alloc("d1", "", 0, "", 0); - disk2 = disk_alloc("d2", "", 0, "", 0); - tommy_list_insert_tail(&state.disklist, &disk1->node, disk1); - tommy_list_insert_tail(&state.disklist, &disk2->node, disk2); - - /* initially both disks are empty */ - if (fs_count_unsynced(&state, 10, 100) != 0) { - log_fatal(EINTERNAL, "test_unsynced_sweep: empty disks failed\n"); - exit(EXIT_FAILURE); - } - - /* file1 on disk1: 10 blocks at parity 10..19 (first 5 BLK, next 5 CHG) */ - file1 = file_alloc(4096, "f1", 10 * 4096, 0, 0, 0); - for (i = 0; i < 5; ++i) - block_state_set(file_block(file1, i), BLOCK_STATE_BLK); - for (i = 5; i < 10; ++i) - block_state_set(file_block(file1, i), BLOCK_STATE_CHG); - fs_allocate(disk1, 10, file1, 0, 10); - - /* file2 on disk2: 20 blocks at parity 0..19 (all BLK) */ - file2 = file_alloc(4096, "f2", 20 * 4096, 0, 0, 0); - for (i = 0; i < 20; ++i) - block_state_set(file_block(file2, i), BLOCK_STATE_BLK); - fs_allocate(disk2, 0, file2, 0, 20); - - /* parity 0..14 has only BLK or EMPTY; parity 15..19 has CHG on disk1 and BLK on disk2 -> 5 unsynced */ - if (fs_count_unsynced(&state, 15, 20) != 5) { - log_fatal(EINTERNAL, "test_unsynced_sweep: mixed block run failed\n"); - exit(EXIT_FAILURE); - } - - /* deleted run on disk1: 3 blocks at parity 20..22 */ - del = file_alloc(4096, "", 3 * 4096, 0, 0, 0); - for (i = 0; i < 3; ++i) - block_state_set(file_block(del, i), BLOCK_STATE_DELETED); - fs_allocate(disk1, 20, del, 0, 3); - - /* disk2 is empty at 20..22 -> DELETED alone does not count as unsynced */ - if (fs_count_unsynced(&state, 15, 23) != 5) { - log_fatal(EINTERNAL, "test_unsynced_sweep: deleted without other file failed\n"); - exit(EXIT_FAILURE); - } - - /* file3 on disk2: first 2 blocks at parity 21..22 (BLK) */ - file3 = file_alloc(4096, "f3", 6 * 4096, 0, 0, 0); - for (i = 0; i < 2; ++i) - block_state_set(file_block(file3, i), BLOCK_STATE_BLK); - block_state_set(file_block(file3, 2), BLOCK_STATE_REP); - block_state_set(file_block(file3, 3), BLOCK_STATE_REP); - block_state_set(file_block(file3, 4), BLOCK_STATE_BLK); - block_state_set(file_block(file3, 5), BLOCK_STATE_REBUILD); - fs_allocate(disk2, 21, file3, 0, 2); - - /* parity 20: DELETED on d1, empty on d2 -> not unsynced; - * parity 21..22: DELETED on d1, BLK on d2 -> unsynced (2 blocks). Total 5 + 2 = 7 */ - if (fs_count_unsynced(&state, 15, 23) != 7) { - log_fatal(EINTERNAL, "test_unsynced_sweep: deleted with file failed\n"); - exit(EXIT_FAILURE); - } - - /* the second extent of file3 starts at a nonzero file offset after a gap */ - fs_allocate(disk2, 30, file3, 2, 4); - - /* file4 shares extent boundaries with file3 and changes flags at 32 and 33 */ - file4 = file_alloc(4096, "f4", 4 * 4096, 0, 0, 0); - for (i = 0; i < 4; ++i) - block_state_set(file_block(file4, i), BLOCK_STATE_BLK); - block_state_set(file_block(file4, 2), BLOCK_STATE_CHG); - fs_allocate(disk1, 30, file4, 0, 4); - - /* - * Positions 30..33 are all unsynced: REP, REP, CHG, REBUILD respectively. - * At 32 and 33 the invalid block changes disks at the same event boundary; - * count each position once using the refreshed flags from both disks. - * Check every window between limits in gaps, within extents and runs, - * and at their boundaries, including empty windows and nonzero starts. - */ - for (i = 0; i < sizeof(bounds) / sizeof(bounds[0]); ++i) { - block_off_t j; - for (j = 0; j <= i; ++j) { - uint64_t expected = bounds[i].unsynced - bounds[j].unsynced; - uint64_t count = fs_count_unsynced(&state, bounds[j].blockmax, bounds[i].blockmax); - if (count != expected) { - log_fatal(EINTERNAL, "test_unsynced_sweep: window [%" PRIu64 ", %" PRIu64 ") expected %" PRIu64 ", got %" PRIu64 "\n", - bounds[j].blockmax, bounds[i].blockmax, expected, count); - exit(EXIT_FAILURE); - } - } - } - - file_free(file1); - file_free(file2); - file_free(file3); - file_free(file4); - file_free(del); - disk_free(disk1); - disk_free(disk2); -} - static void test_raid(void) { /* vandermonde raid parity generation with 32 data disks */ @@ -3105,9 +2941,6 @@ void test(int argc, char* argv[]) /* extent-aware file-to-parity run lookup */ test_extent_run(); - /* extent sweep for unsynced parity blocks */ - test_unsynced_sweep(); - /* wildcard and path pattern matching */ test_wnmatch(); diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 598a90fe..2d806fb9 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -311,152 +311,152 @@ static struct option long_options[] = { { "gui-threshold-removes", 1, 0, OPT_GUI_THRESHOLD_REMOVES }, /* undocumented GUI, abort sync if too many removed files */ { "gui-threshold-updates", 1, 0, OPT_GUI_THRESHOLD_UPDATES }, /* undocumented GUI, abort sync if too many updated files */ - /* the following are test specific options, DO NOT USE! */ + /* The following are test specific options, DO NOT USE! */ - /* after syncing, do not write the new content file */ + /* After syncing, do not write the new content file */ { "test-kill-after-sync", 0, 0, OPT_TEST_KILL_AFTER_SYNC }, - /* do not execute syncing after resizing parity files, before the normal post-resize content write */ + /* Do not execute syncing after resizing parity files, before the normal post-resize content write */ { "test-kill-after-resize", 0, 0, OPT_TEST_KILL_AFTER_RESIZE }, - /* do not execute syncing after writing the new initial content file */ + /* Do not execute syncing after writing the new initial content file */ { "test-kill-before-sync", 0, 0, OPT_TEST_KILL_BEFORE_SYNC }, - /* exit with failure if after check/fix there ARE NOT unrecoverable errors. */ + /* Exit with failure if after check/fix there ARE NOT unrecoverable errors. */ { "test-expect-unrecoverable", 0, 0, OPT_TEST_EXPECT_UNRECOVERABLE }, - /* exit with failure if after check/fix there ARE NOT recoverable errors. */ + /* Exit with failure if after check/fix there ARE NOT recoverable errors. */ { "test-expect-recoverable", 0, 0, OPT_TEST_EXPECT_RECOVERABLE }, - /* skip the initial self test */ + /* Skip the initial self test */ { "test-skip-self", 0, 0, OPT_TEST_SKIP_SELF }, - /* skip the initial sign check when reading the content file */ + /* Skip the initial sign check when reading the content file */ { "test-skip-sign", 0, 0, OPT_TEST_SKIP_SIGN }, - /* skip the fallocate() when growing the parity files */ + /* Skip the fallocate() when growing the parity files */ { "test-skip-fallocate", 0, 0, OPT_TEST_SKIP_FALLOCATE }, - /* skip the device check */ + /* Skip the device check */ { "test-skip-device", 0, 0, OPT_TEST_SKIP_DEVICE }, - /* force Murmur3 hash */ + /* Force Murmur3 hash */ { "test-force-murmur3", 0, 0, OPT_TEST_FORCE_MURMUR3 }, - /* force Spooky2 hash */ + /* Force Spooky2 hash */ { "test-force-spooky2", 0, 0, OPT_TEST_FORCE_SPOOKY2 }, - /* skip the use of lock file */ + /* Skip the use of lock file */ { "test-skip-lock", 0, 0, OPT_TEST_SKIP_LOCK }, - /* force a sort order for files */ + /* Force a sort order for files */ { "test-force-order-inode", 0, 0, OPT_TEST_FORCE_ORDER_INODE }, { "test-force-order-alpha", 0, 0, OPT_TEST_FORCE_ORDER_ALPHA }, { "test-force-order-dir", 0, 0, OPT_TEST_FORCE_ORDER_DIR }, - /* force scrub of the specified number of blocks */ + /* Force scrub of the specified number of blocks */ { "test-force-scrub-at", 1, 0, OPT_TEST_FORCE_SCRUB_AT }, - /* force scrub of all the even blocks. This is really for testing, don't try it */ + /* Force scrub of all the even blocks. This is really for testing, don't try it */ { "test-force-scrub-even", 0, 0, OPT_TEST_FORCE_SCRUB_EVEN }, - /* force write of the content file even if no modification is done */ + /* Force write of the content file even if no modification is done */ { "test-force-content-write", 0, 0, OPT_TEST_FORCE_CONTENT_WRITE }, - /* relax the checks done at the content file */ + /* Relax the checks done at the content file */ { "test-skip-content-check", 0, 0, OPT_TEST_SKIP_CONTENT_CHECK }, - /* skip the parity access */ + /* Skip the parity access */ { "test-skip-parity-access", 0, 0, OPT_TEST_SKIP_PARITY_ACCESS }, - /* exit generic failure */ + /* Exit generic failure */ { "test-expect-failure", 0, 0, OPT_TEST_EXPECT_FAILURE }, - /* exit generic need sync */ + /* Exit generic need sync */ { "test-expect-need-sync", 0, 0, OPT_TEST_EXPECT_NEED_SYNC }, #if HAVE_CHECKER - /* run some command after loading the state and before the command */ + /* Run some command after loading the state and before the command */ { "test-run", 1, 0, OPT_TEST_RUN }, #endif - /* use the FindFirst/Next approach in Windows to list files */ + /* Use the FindFirst/Next approach in Windows to list files */ { "test-force-scan-winfind", 0, 0, OPT_TEST_FORCE_SCAN_WINFIND }, - /* alternative import working by data */ + /* Alternative import working by data */ { "test-import-content", 1, 0, OPT_TEST_IMPORT_CONTENT }, - /* force immediate progress state update */ + /* Force immediate progress state update */ { "test-force-progress", 0, 0, OPT_TEST_FORCE_PROGRESS }, - /* skip the disk access */ + /* Skip the disk access */ { "test-skip-disk-access", 0, 0, OPT_TEST_SKIP_DISK_ACCESS }, - /* force autosave at the specified block */ + /* Force autosave at the specified block */ { "test-force-autosave-at", 1, 0, OPT_TEST_FORCE_AUTOSAVE_AT }, - /* fake device data */ + /* Fake device data */ { "test-fake-device", 0, 0, OPT_TEST_FAKE_DEVICE }, - /* fake UUID */ + /* Fake UUID */ { "test-fake-uuid", 0, 0, OPT_TEST_FAKE_UUID }, - /* match first UUID */ + /* Match first UUID */ { "test-match-first-uuid", 0, 0, OPT_TEST_MATCH_FIRST_UUID }, - /* force parity update even if all the data hash is already matching */ + /* Force parity update even if all the data hash is already matching */ { "test-force-parity-update", 0, 0, OPT_TEST_FORCE_PARITY_UPDATE }, - /* number of IO buffers */ + /* Number of IO buffers */ { "test-io-cache", 1, 0, OPT_TEST_IO_CACHE }, - /* print IO stats */ + /* Print IO stats */ { "test-io-stats", 0, 0, OPT_TEST_IO_STATS }, /* now replaced by -A, --stats */ - /* signal condition variable outside the mutex */ + /* Signal condition variable outside the mutex */ { "test-cond-signal-outside", 0, 0, OPT_TEST_COND_SIGNAL_OUTSIDE }, - /* set the io advise to none */ + /* Set the io advise to none */ { "test-io-advise-none", 0, 0, OPT_TEST_IO_ADVISE_NONE }, - /* set the io advise to sequential */ + /* Set the io advise to sequential */ { "test-io-advise-sequential", 0, 0, OPT_TEST_IO_ADVISE_SEQUENTIAL }, - /* set the io advise to flush */ + /* Set the io advise to flush */ { "test-io-advise-flush", 0, 0, OPT_TEST_IO_ADVISE_FLUSH }, - /* set the io advise to flush window */ + /* Set the io advise to flush window */ { "test-io-advise-flush-window", 0, 0, OPT_TEST_IO_ADVISE_FLUSH_WINDOW }, - /* set the io advise to discard */ + /* Set the io advise to discard */ { "test-io-advise-discard", 0, 0, OPT_TEST_IO_ADVISE_DISCARD }, - /* set the io advise to discard window */ + /* Set the io advise to discard window */ { "test-io-advise-discard-window", 0, 0, OPT_TEST_IO_ADVISE_DISCARD_WINDOW }, - /* set the io advise to direct */ + /* Set the io advise to direct */ { "test-io-advise-direct", 0, 0, OPT_TEST_IO_ADVISE_DIRECT }, - /* set an artificial parity limit */ + /* Set an artificial parity limit */ { "test-parity-limit", 1, 0, OPT_TEST_PARITY_LIMIT }, - /* skip content write */ + /* Skip content write */ { "test-skip-content-write", 0, 0, OPT_TEST_SKIP_CONTENT_WRITE }, - /* skip space holder file in parity disks */ + /* Skip space holder file in parity disks */ { "test-skip-space-holder", 0, 0, OPT_TEST_SKIP_SPACE_HOLDER }, - /* set the output format */ + /* Set the output format */ { "test-fmt", 1, 0, OPT_TEST_FORMAT }, - /* skip thread in disk scan */ + /* Skip thread in disk scan */ { "test-skip-multi-scan", 0, 0, OPT_TEST_SKIP_MULTI_SCAN }, { 0, 0, 0, 0 } }; #endif -/** +/* * Free letters: gIjJkKMnPQruxXz * * The 's' letter is used in main.c @@ -1530,8 +1530,7 @@ int snapraid_main(int argc, char* argv[]) if (state.opt.force_realloc) state_locate_mark_tail_blocks_for_resync(&state, opt.parity_tail); - /* skip touch itself until content is loaded and sync has initialized parity sizes */ - if (opt.gui_touch_before && state_is_write_ready(&state)) { + if (opt.gui_touch_before) { ret = state_touch(&state); /* save the new state if required */ @@ -1628,9 +1627,6 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_REHASH) { state_read(&state); - /* prevent saving content with uninitialized parity split sizes */ - state_require_write_ready(&state); - state_rehash(&state); /* @@ -1649,9 +1645,6 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_SCRUB) { state_read(&state); - /* prevent saving content with uninitialized parity split sizes */ - state_require_write_ready(&state); - /* refresh the size info before the content write */ state_refresh(&state); @@ -1674,9 +1667,6 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_REWRITE) { state_read(&state); - /* prevent saving content with uninitialized parity split sizes */ - state_require_write_ready(&state); - /* intercept signals while operating */ os_signal_init(app_signal_handler, app_signal_handler); @@ -1692,9 +1682,6 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_TOUCH) { state_read(&state); - /* prevent saving content with uninitialized parity split sizes */ - state_require_write_ready(&state); - memory(); ret = state_touch(&state); @@ -1709,7 +1696,6 @@ int snapraid_main(int argc, char* argv[]) */ os_signal_init(app_signal_handler, app_signal_handler); - /* save the new state if required */ if (state.need_write) state_write(&state); } else if (operation == OPERATION_SPINUP) { diff --git a/cmdline/snapraid.h b/cmdline/snapraid.h index 2b2c9eb4..3bff83ec 100644 --- a/cmdline/snapraid.h +++ b/cmdline/snapraid.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef SNAPRAID_H -#define SNAPRAID_H +#ifndef __SNAPRAID_H +#define __SNAPRAID_H /****************************************************************************/ /* snapraid */ diff --git a/cmdline/speed.c b/cmdline/speed.c index 7514ed8a..c690986e 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -11,11 +11,18 @@ #include "raid/memory.h" #include "state.h" -#if HAVE_ASSEMBLY -#define SPEED_BARRIER() __asm__ __volatile__ ("" : : : "memory") -#else -#define SPEED_BARRIER() do { } while (0) -#endif +/** + * Differential us of two timeval. + */ +static int64_t diffgettimeofday(struct timeval* start, struct timeval* stop) +{ + int64_t d; + + d = 1000000LL * (stop->tv_sec - start->tv_sec); + d += stop->tv_usec - start->tv_usec; + + return d; +} #define SPEED_START \ { \ @@ -23,17 +30,15 @@ int64_t elapsed_ = 0; \ int64_t best_dt_ = INT64_MAX; \ do { \ - SPEED_BARRIER(); \ - uint64_t start_ = os_tick_ns(); \ - SPEED_BARRIER(); \ + struct timeval start; \ + struct timeval stop; \ + gettimeofday(&start, 0); \ for (int i_ = 0; i_ < speed_delta_; ++i_) #define SPEED_STOP \ - SPEED_BARRIER(); \ - uint64_t stop_ = os_tick_ns(); \ - SPEED_BARRIER(); \ - dt = stop_ > start_ ? stop_ - start_ : 0; \ - if (dt < 1000000 && speed_delta_ <= INT_MAX / 2) { \ + gettimeofday(&stop, 0); \ + dt = diffgettimeofday(&start, &stop); \ + if (dt < 1000 && speed_delta_ <= INT_MAX / 2) { \ speed_delta_ *= 2; \ elapsed_ = 0; \ best_dt_ = INT64_MAX; \ @@ -42,8 +47,8 @@ elapsed_ += dt; \ if (dt < best_dt_) \ best_dt_ = dt; \ - } while (elapsed_ < period * 1000000LL); \ - ds = size * (int64_t)speed_delta_ * nd * 1000LL; \ + } while (elapsed_ < period * 1000LL); \ + ds = size * (int64_t)speed_delta_ * nd; \ dt = best_dt_; \ } diff --git a/cmdline/spooky2.c b/cmdline/spooky2.c index 67a86efe..ca063a32 100644 --- a/cmdline/spooky2.c +++ b/cmdline/spooky2.c @@ -96,7 +96,7 @@ // #define sc_const 0xdeadbeefdeadbeefULL -static void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) +void SpookyHash128(const void* data, size_t size, const uint8_t* seed, uint8_t* digest) { uint64_t h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11; uint8_t buf[sc_blockSize]; @@ -117,11 +117,6 @@ static void SpookyHash128(const void* data, size_t size, const uint8_t* seed, ui end = p + nblocks * sc_blockSize; /* body */ -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC unroll 1 -#elif defined(__clang__) -#pragma unroll 1 -#endif while (p < end) { Mix(p, h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11); p += sc_blockSize; diff --git a/cmdline/state.c b/cmdline/state.c index 006a912d..c3798beb 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -32,7 +32,7 @@ #define HAVE_MT_VERIFY 1 #endif -/** +/* * Start content-file writeback in bounded windows to avoid dirty-page * throttling; the subsequent fsync() remains responsible for durability. * @@ -171,6 +171,7 @@ void state_init(struct snapraid_state* state) state->autosave = 0; state->need_write = 0; state->written = 0; + state->checked_read = 0; state->block_size = 256 * KIBI; /* default 256 KiB */ state->raid_mode = RAID_MODE_CAUCHY_RAID; state->file_mode = ADVISE_DEFAULT; @@ -2196,146 +2197,31 @@ static inline int fs_is_block_deleted(struct snapraid_disk* disk, block_off_t po } /** - * Advance a disk cursor to pos and cache its flags and next event boundary. - * Events occur at extent boundaries and changes in block flags within an extent. + * Check if a block parity needs sync + * + * This is the same logic used in "status" to detect an incomplete "sync", + * that ignores invalid block, if they are not used by a file in any disk. + * + * This means that DELETED blocks won't necessarily imply an invalid parity. */ -static inline void disk_advance( - tommy_tree_node** cursor_ptr, - block_off_t pos, - block_off_t blockmax, - int* has_file_ptr, - int* has_invalid_ptr, - block_off_t* next_event_ptr, - size_t stride -) +static int fs_is_block_unsynced(struct snapraid_state* state, block_off_t pos) { - /* advance cursor if the current extent ends at or before pos */ - while (*cursor_ptr != 0) { - struct snapraid_extent* ext = (*cursor_ptr)->data; - if (ext->parity_pos + ext->count > pos) - break; - *cursor_ptr = tommy_tree_next(*cursor_ptr); + int one_invalid = 0; + int one_valid = 0; + for (tommy_node* i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + struct snapraid_block* block = fs_par2block_find(disk, pos); + + if (block_has_file(block)) + one_valid = 1; + if (block_has_invalid_parity(block)) + one_invalid = 1; + + if (one_invalid && one_valid) + return 1; } - if (*cursor_ptr == 0) { - /* no more extents on this disk */ - *has_file_ptr = 0; - *has_invalid_ptr = 0; - *next_event_ptr = blockmax; - return; - } - - struct snapraid_extent* ext = (*cursor_ptr)->data; - if (ext->parity_pos > pos) { - /* in a gap before the next extent */ - *has_file_ptr = 0; - *has_invalid_ptr = 0; - *next_event_ptr = ext->parity_pos; - } else { - /* pos is inside this extent */ - block_off_t extent_rem = ext->parity_pos + ext->count - pos; - block_off_t file_pos = ext->file_pos + (pos - ext->parity_pos); - struct snapraid_block* block = file_block(ext->file, file_pos); - int has_file = block_has_file(block); - int has_invalid = block_has_invalid_parity(block); - block_off_t run_rem = 1; - const unsigned char* ptr = (const unsigned char*)block; - - /* stop inspecting block flags at the end of the requested window */ - if (extent_rem > blockmax - pos) - extent_rem = blockmax - pos; - - *has_file_ptr = has_file; - *has_invalid_ptr = has_invalid; - - /* find how many consecutive blocks on this disk share the same flags */ - while (run_rem < extent_rem) { - const struct snapraid_block* next_b = (const struct snapraid_block*)(ptr + run_rem * stride); - if (block_has_file(next_b) != has_file || block_has_invalid_parity(next_b) != has_invalid) - break; - ++run_rem; - } - - *next_event_ptr = pos + run_rem; - } -} - -/** - * Count the number of unsynced parity positions using an event sweep across extents. - * - * An unsynced parity position is one where at least one disk has a live file - * block, and at least one disk has an invalid parity block (CHG, REP, REBUILD, - * or DELETED). - * - * Counts only positions in [blockmin, blockmax). - * The sweep takes O(D * log(E) + B + R * D) time, where E is the number of - * extents per disk, B is the number of blocks inspected within the window, - * R is the number of event positions processed (including initialization), and - * D is the number of disks. Cached event boundaries avoid rescanning block runs. - * - * The caller must ensure that the disk list, extent trees, and block states - * remain unchanged during the count, as no locks are held during the sweep. - */ -uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmin, block_off_t blockmax) -{ - tommy_tree_node* cursors[RAID_DATA_MAX]; - block_off_t next_events[RAID_DATA_MAX]; - int disk_has_file[RAID_DATA_MAX]; - int disk_has_invalid[RAID_DATA_MAX]; - unsigned disk_count = 0; - tommy_node* node; - uint64_t total_unsynced = 0; - block_off_t pos = blockmin; - size_t stride = block_sizeof(); - unsigned d; - - /* state_map() validates the disk limit only after content loading */ - if (tommy_list_count(&state->disklist) > RAID_DATA_MAX) { - /* LCOV_EXCL_START */ - log_fatal(ESOFT, "Too many data disks. Maximum allowed is %u.\n", RAID_DATA_MAX); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - for (node = state->disklist; node != 0; node = node->next) { - struct snapraid_disk* disk = node->data; - struct snapraid_extent* ext = fs_par2extent_find_next(disk, blockmin); - cursors[disk_count] = ext ? &ext->parity_node : 0; - /* force an initial event so flags are set before their first use */ - next_events[disk_count] = blockmin; - ++disk_count; - } - - while (pos < blockmax) { - block_off_t next_pos = blockmax; - int one_file = 0; - int one_invalid = 0; - - for (d = 0; d < disk_count; ++d) { - /* refresh expired runs before counting the interval starting at pos */ - if (next_events[d] == pos) { - disk_advance( - &cursors[d], pos, blockmax, - &disk_has_file[d], &disk_has_invalid[d], - &next_events[d], stride - ); - } - - if (disk_has_file[d]) - one_file = 1; - if (disk_has_invalid[d]) - one_invalid = 1; - if (next_events[d] < next_pos) - next_pos = next_events[d]; - } - - if (one_file && one_invalid) - total_unsynced += (next_pos - pos); - - pos = next_pos; - } - - return total_unsynced; + return 0; } /** @@ -2408,15 +2294,14 @@ static void decoding_error(const char* path, STREAM* f) * pointer directly over cached hashes therefore preserves the same integrity * check performed by sread(). */ -__aligned_loops(64) static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count, unsigned state, int discard_hash) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); - size_t hash_size = BLOCK_HASH_SIZE; + size_t block_stride = block_sizeof(); while (count) { size_t available = (size_t)(f->end - f->pos); - block_off_t cached_count = available / hash_size; + block_off_t cached_count = available / BLOCK_HASH_SIZE; unsigned char* input; if (cached_count > count) @@ -2427,50 +2312,33 @@ static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off int ret; block_state_set(block, state); - ret = sread(f, block->hash, hash_size); + ret = sread(f, block->hash, BLOCK_HASH_SIZE); if (ret < 0) return -1; if (discard_hash) hash_invalid_set(block->hash); - block_ptr += 1 + hash_size; + block_ptr += block_stride; --count; continue; } - count -= cached_count; input = f->pos; - if (hash_size == HASH_MAX) { - while (cached_count != 0) { - struct snapraid_block* block = (struct snapraid_block*)block_ptr; + for (block_off_t i = 0; i < cached_count; ++i) { + struct snapraid_block* block = (struct snapraid_block*)block_ptr; - block_state_set(block, state); - if (discard_hash) - memset(block->hash, 0, HASH_MAX); - else - memcpy(block->hash, input, HASH_MAX); + block_state_set(block, state); + if (discard_hash) + hash_invalid_set(block->hash); + else + hash_copy(block->hash, input); - input += HASH_MAX; - block_ptr += 1 + HASH_MAX; - --cached_count; - } - } else { - while (cached_count != 0) { - struct snapraid_block* block = (struct snapraid_block*)block_ptr; - - block_state_set(block, state); - if (discard_hash) - hash_invalid_set(block->hash); - else - hash_copy(block->hash, input); - - input += hash_size; - block_ptr += 1 + hash_size; - --cached_count; - } + input += BLOCK_HASH_SIZE; + block_ptr += block_stride; } f->pos = input; + count -= cached_count; } return 0; @@ -2483,8 +2351,6 @@ static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off */ static void fs_single_thread(struct snapraid_state* state, int single_thread) { - memory_single_thread(single_thread); - #if HAVE_THREAD for (tommy_node* i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; @@ -2492,6 +2358,7 @@ static void fs_single_thread(struct snapraid_state* state, int single_thread) } #else (void)state; + (void)single_thread; #endif } @@ -2514,8 +2381,6 @@ struct state_read_context { static void state_read_content(struct snapraid_state* state, const char* path, STREAM* f, struct state_read_context* context) { block_off_t blockmax; - block_off_t inv_min; - block_off_t inv_max; uint64_t count_file; uint64_t count_hardlink; uint64_t count_symlink; @@ -2531,16 +2396,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S int has_hash; int has_prevhash; int has_info; + int has_invalid_parity; char buffer[PATH_MAX]; - ssize_t ret; + int ret; tommy_array disk_mapping; uint32_t mapping_max; - tommy_size_t disk_file_count[RAID_DATA_MAX]; tommy_hashdyn bucket_hash; blockmax = 0; - inv_min = BLOCK_MAX; - inv_max = 0; count_file = 0; count_hardlink = 0; count_symlink = 0; @@ -2556,8 +2419,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S has_hash = 0; has_prevhash = 0; has_info = 0; + has_invalid_parity = 0; mapping_max = 0; - memset(disk_file_count, 0, sizeof(disk_file_count)); tommy_array_init(&disk_mapping); tommy_hashdyn_init(&bucket_hash); @@ -2565,7 +2428,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid header: read failed!\n"); + log_fatal(ECONTENT, "Invalid header!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2594,7 +2457,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_START */ if (memcmp(buffer, "SNAPCNT", 7) != 0) { decoding_error(path, f); - log_fatal(ECONTENT, "Invalid header: magic mismatch!\n"); + log_fatal(ECONTENT, "Invalid header!\n"); os_abort(); } else { log_fatal(ECONTENT, "The content file '%s' was generated with a newer version of SnapRAID!\n", path); @@ -2628,7 +2491,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: File mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2638,7 +2501,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2647,23 +2509,23 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'f' size %" PRIu64 " exceeds maximum!\n", v_size); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Invalid file size %" PRIu64 "!\n", v_size); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } if (state->block_size == 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' encountered with zero blocksize\n"); - os_abort(); + log_fatal(EINTERNAL, "Internal inconsistency: Zero blocksize\n"); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } if (!has_blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' missing 'blockmax' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Missing 'blockmax' before file in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2672,7 +2534,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size / state->block_size > blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' size is too big!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: File size too big!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2681,27 +2543,20 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid mtime_sec\n"); os_abort(); /* LCOV_EXCL_STOP */ } ret = sgetb32(f, &v_mtime_nsec); - if (ret < 0) { + if (ret < 0 || v_mtime_nsec > 1000000000) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid mtime_nsec\n"); os_abort(); /* LCOV_EXCL_STOP */ } - /* - * STAT_NSEC_INVALID is encoded as 0. - * Values > 1000000000 can occur in content files written by SnapRAID <= 14.10 - * on Windows for files with pre-1970 timestamps due to negative nanoseconds. - * Normalize them to STAT_NSEC_INVALID. - */ - if (v_mtime_nsec == 0 || v_mtime_nsec > 1000000000) + /* STAT_NSEC_INVALID is encoded as 0 */ + if (v_mtime_nsec == 0) v_mtime_nsec = STAT_NSEC_INVALID; else --v_mtime_nsec; @@ -2710,27 +2565,21 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid inode\n"); os_abort(); /* LCOV_EXCL_STOP */ } - size_t sub_len; - ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } - sub_len = ret; - if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' has invalid subpath '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Invalid file '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2739,18 +2588,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S * Allocate without initializing blocks. The runs below cover the whole * file and set every state and hash before exposing the loaded state. */ - file = file_alloc_len(state->block_size, sub, sub_len, v_size, v_mtime_sec, v_mtime_nsec, v_inode); + file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode); - /* - * Append the file to the disk list. Hash table insertions into - * inodeset, pathset, and stampset are deferred until all records - * are parsed to avoid cache thrashing and allow pre-reserving buckets. - * Cache the path hash in pathset.index to avoid recomputing strlen - * during insertion. - */ - file->pathset.index = file_path_hash_len(file->sub, sub_len); + /* insert the file in the file containers */ + if (file->inode != INODE_INVALID) + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); - ++disk_file_count[mapping]; /* read all the blocks */ v_idx = 0; @@ -2768,7 +2613,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block run has an invalid pos\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2777,7 +2621,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block run has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2785,7 +2628,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count == 0 || v_count > file->blockmax || v_idx > file->blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block number out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Block number out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2793,7 +2636,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count > blockmax || v_pos > blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' block position out of range %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); + log_fatal(EINTERNAL, "Internal inconsistency: Block size %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2821,25 +2664,20 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'f' has invalid block type '%c'!\n", (char)c); + log_fatal(ECONTENT, "Invalid block type!\n"); os_abort(); /* LCOV_EXCL_STOP */ } /* all file block states except BLK require a parity update */ - if (v_state != BLOCK_STATE_BLK) { - if (v_pos < inv_min) - inv_min = v_pos; - if (v_pos + v_count > inv_max) - inv_max = v_pos + v_count; - } + if (v_state != BLOCK_STATE_BLK) + has_invalid_parity = 1; v_file_pos = v_idx; ret = state_read_block_run(f, file, v_file_pos, v_count, v_state, discard_hash); if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'f' error reading block run\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2860,7 +2698,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_info || !has_blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' duplicate or misplaced in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'info' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2870,7 +2708,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' has an invalid oldest timestamp\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2879,8 +2716,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_oldest > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'i' has an out-of-range oldest timestamp %" PRIu64 "!\n", v_oldest); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Invalid info oldest timestamp %" PRIu64 "!\n", v_oldest); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -2897,7 +2734,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' run has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2905,7 +2741,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count > blockmax || v_pos > blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' run size out of range %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); + log_fatal(EINTERNAL, "Internal inconsistency: Info size %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2914,7 +2750,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' has invalid flags\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2926,7 +2761,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' has an invalid timestamp\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2935,8 +2769,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (t64 > INT64_MAX - v_oldest) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'i' has an overflowing timestamp %" PRIu64 "!\n", t64); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Invalid info timestamp %" PRIu64 "!\n", t64); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -2955,7 +2789,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (rehash && state->prevhash == HASH_UNDEFINED) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' requires previous checksum but none defined\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Missing previous checksum!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2964,7 +2798,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!info) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' failed to create info object\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Missing info!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -2975,22 +2809,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S info_set_run(&state->infoarr, v_pos, v_count, info); v_pos += v_count; } else { - /* - * This branch is entered only when a parity position below blockmax - * has no info, which means that all data disks simultaneously have a - * hole (no file block) at this exact parity position. - * Since blockmax is the highest allocated block across all disks and - * file allocation on disks is typically contiguous from block 0, such - * global parity holes are extremely rare in practice. Even though the - * block-by-block fs_info_is_required() validation is inefficient, it is - * practically never executed, or only for very short gaps. - */ while (v_count) { /* ensure that an info is present only for used positions */ if (fs_info_is_required(state, v_pos)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'i' missing required info for position %" PRIu64 "\n", (uint64_t)v_pos); + log_fatal(EINTERNAL, "Internal inconsistency: Missing info!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3001,6 +2825,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } } } + + /* + * A synchronized content has no invalid block state, so its unsynced + * count is certainly zero. Scan the extent maps only when needed. + */ + if (has_invalid_parity) { + for (v_pos = 0; v_pos < blockmax; ++v_pos) + if (fs_is_block_unsynced(state, v_pos)) + ++count_unsynced; + } } else if (c == 'h') { /* hole */ block_off_t v_pos; @@ -3011,7 +2845,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Hole mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3020,7 +2854,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!has_blockmax) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' missing 'blockmax' in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Missing 'blockmax' before hole in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3034,7 +2868,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' run has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3042,7 +2875,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_count == 0 || v_count > blockmax || v_pos > blockmax - v_count) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' size out of range %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); + log_fatal(EINTERNAL, "Internal inconsistency: Hole size %" PRIu64 "/%" PRIu64 "!\n", blockmax, v_pos + v_count); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3053,17 +2886,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S switch (c) { case 'o' : /* if it's a run of deleted blocks */ - if (v_pos < inv_min) - inv_min = v_pos; - if (v_pos + v_count > inv_max) - inv_max = v_pos + v_count; + has_invalid_parity = 1; /* ensure the converted byte size fits in signed data_off_t without overflow */ if (v_count > INT64_MAX / (uint64_t)state->block_size) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'h' deleted block count %" PRIu64 " overflows size!\n", v_count); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Invalid deleted block count %" PRIu64 "!\n", v_count); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3084,7 +2914,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'h' error reading deleted block run\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3100,7 +2929,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'h' has invalid hole type '%c'!\n", (char)c); + log_fatal(ECONTENT, "Invalid hole type!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3116,7 +2945,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Dealloc mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3126,7 +2955,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' has an invalid count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3139,22 +2967,17 @@ static void state_read_content(struct snapraid_state* state, const char* path, S uint64_t v_mtime_sec; uint32_t v_mtime_nsec; - size_t sub_len; - ret = sgetbs(f, sub, sizeof(sub)); if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } - sub_len = ret; - if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has invalid subpath '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Invalid dealloc '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3163,7 +2986,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3172,8 +2994,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'd' entry has an out-of-range size %" PRIu64 "!\n", v_size); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Invalid deallocated file size %" PRIu64 "!\n", v_size); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3181,33 +3003,26 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid mtime_sec\n"); os_abort(); /* LCOV_EXCL_STOP */ } ret = sgetb32(f, &v_mtime_nsec); - if (ret < 0) { + if (ret < 0 || v_mtime_nsec > 1000000000) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' entry has an invalid mtime_nsec\n"); os_abort(); /* LCOV_EXCL_STOP */ } - /* - * STAT_NSEC_INVALID is encoded as 0. - * Values > 1000000000 can occur in content files written by SnapRAID <= 14.10 - * on Windows for files with pre-1970 timestamps due to negative nanoseconds. - * Normalize them to STAT_NSEC_INVALID. - */ - if (v_mtime_nsec == 0 || v_mtime_nsec > 1000000000) + /* STAT_NSEC_INVALID is encoded as 0 */ + if (v_mtime_nsec == 0) v_mtime_nsec = STAT_NSEC_INVALID; else --v_mtime_nsec; /* allocate the file */ - struct snapraid_dealloc* dealloc = dealloc_alloc_len(state->block_size, sub, sub_len, v_size, v_mtime_sec, v_mtime_nsec); + struct snapraid_dealloc* dealloc = dealloc_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec); log_tag("content_info:dealloc_entry:%s:%s:%" PRIu64 ":%" PRIu64 ":%u\n", disk->name, esc_tag(dealloc->sub), dealloc->size, dealloc->mtime_sec, dealloc->mtime_nsec); @@ -3222,7 +3037,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (state->prevhash == HASH_UNDEFINED) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' previous hash without previous checksum\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Previous dealloc hash without previous checksum!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3230,7 +3045,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } else { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'd' has invalid hash type '%c'!\n", (char)c); + log_fatal(ECONTENT, "Invalid dealloc hash type!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3239,7 +3054,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'd' error reading block hash\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3252,8 +3066,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* symlink */ char sub[PATH_MAX]; char linkto[PATH_MAX]; - size_t sub_len; - size_t linkto_len; struct snapraid_link* slink; struct snapraid_disk* disk; uint32_t mapping; @@ -3262,7 +3074,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 's' mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Symlink mapping index out of range\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3272,16 +3084,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 's' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } - sub_len = ret; if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 's' has invalid subpath '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Invalid symlink '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3290,17 +3100,15 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 's' has an invalid target path\n"); os_abort(); /* LCOV_EXCL_STOP */ } - linkto_len = ret; /* allocate the link as symbolic link */ - slink = link_alloc_len(sub, sub_len, linkto, linkto_len, FILE_IS_SYMLINK); + slink = link_alloc(sub, linkto, FILE_IS_SYMLINK); /* insert the link in the link containers */ - tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash_len(slink->sub, sub_len)); + tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub)); tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink); /* stat */ @@ -3309,8 +3117,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* hardlink */ char sub[PATH_MAX]; char linkto[PATH_MAX]; - size_t sub_len; - size_t linkto_len; struct snapraid_link* slink; struct snapraid_disk* disk; uint32_t mapping; @@ -3319,7 +3125,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Hardlink mapping index out of range!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3329,16 +3135,14 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } - sub_len = ret; if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has invalid subpath '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Invalid hardlink '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3347,25 +3151,23 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has an invalid target path\n"); os_abort(); /* LCOV_EXCL_STOP */ } - linkto_len = ret; if (!path_is_sub(linkto)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'a' has invalid target '%s' for '%s'!\n", linkto, sub); + log_fatal(EINTERNAL, "Internal inconsistency: Invalid hardlink target '%s' for '%s'!\n", linkto, sub); os_abort(); /* LCOV_EXCL_STOP */ } /* allocate the link as hard link */ - slink = link_alloc_len(sub, sub_len, linkto, linkto_len, FILE_IS_HARDLINK); + slink = link_alloc(sub, linkto, FILE_IS_HARDLINK); /* insert the link in the link containers */ - tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash_len(slink->sub, sub_len)); + tommy_hashdyn_insert(&disk->linkset, &slink->nodeset, slink, link_name_hash(slink->sub)); tommy_list_insert_tail(&disk->linklist, &slink->nodelist, slink); /* stat */ @@ -3373,7 +3175,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } else if (c == 'r') { /* dir */ char sub[PATH_MAX]; - size_t sub_len; struct snapraid_dir* dir; struct snapraid_disk* disk; uint32_t mapping; @@ -3382,7 +3183,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0 || mapping >= mapping_max) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'r' mapping index out of range\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Dir mapping index out of range!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3392,25 +3193,23 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'r' has an invalid subpath\n"); os_abort(); /* LCOV_EXCL_STOP */ } - sub_len = ret; if (!path_is_sub(sub)) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'r' has invalid subpath '%s'!\n", sub); + log_fatal(EINTERNAL, "Internal inconsistency: Invalid dir '%s'!\n", sub); os_abort(); /* LCOV_EXCL_STOP */ } /* allocate the dir */ - dir = dir_alloc_len(sub, sub_len); + dir = dir_alloc(sub); /* insert the dir in the dir containers */ - tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash_len(dir->sub, sub_len)); + tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub)); tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir); /* stat */ @@ -3419,7 +3218,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_hash || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'c' duplicate or misplaced the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'hash' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3438,7 +3237,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'c' has invalid checksum type '%c'!\n", (char)c); + log_fatal(ECONTENT, "Invalid checksum!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3448,7 +3247,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'c' error reading hash seed\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3456,7 +3254,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_prevhash || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'C' duplicate or misplaced in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'prevhash' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3475,7 +3273,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S default : /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'C' has invalid checksum type '%c'!\n", (char)c); + log_fatal(ECONTENT, "Invalid checksum!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3485,7 +3283,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'C' error reading hash seed\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3495,7 +3292,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_block_size || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'z' duplicate or misplaced in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'blocksize' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3505,7 +3302,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'z' has an invalid block size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3513,8 +3309,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (block_size == 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'z' has zero 'blocksize' specification in the content file!\n"); - os_abort(); + log_fatal(ECONTENT, "Zero 'blocksize' specification in the content file!\n"); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3537,7 +3333,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_hash_size || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'y' duplicate or misplaced in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'hashsize' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3547,7 +3343,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'y' has an invalid hash size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3563,7 +3358,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (hash_size < 4 || hash_size > HASH_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Invalid 'hashsize' specification %" PRIu64 " in the content file!\n", hash_size); + log_fatal(ECONTENT, "Invalid 'hashsize' specification in the content file!\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3585,7 +3380,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (has_blockmax || mapping_max != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'x' duplicate or misplaced in the content file!\n"); + log_fatal(EINTERNAL, "Internal inconsistency: Duplicate or misplaced 'blockmax' in the content file!\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3595,7 +3390,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'x' has an invalid blockmax\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3630,7 +3424,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid disk name\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3639,7 +3432,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid disk index\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3647,8 +3439,8 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_idx >= RAID_DATA_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'M' invalid disk position '%u' in the content file!\n", v_idx); - os_abort(); + log_fatal(ECONTENT, "Invalid disk position '%u' in the content file!\n", v_idx); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3656,7 +3448,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid total_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3665,7 +3456,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid free_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3675,7 +3465,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'M' has an invalid uuid\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3724,7 +3513,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid parity level\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3733,7 +3521,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid total_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3742,7 +3529,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid free_blocks count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3751,7 +3537,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' has an invalid split count\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3759,16 +3544,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_split_mac > SPLIT_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' has too many splits '%u' in the content file!\n", v_split_mac); - os_abort(); + log_fatal(ECONTENT, "Too many splits '%u' in the content file!\n", v_split_mac); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } if (v_level >= LEV_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' has invalid parity level '%u' in the content file!\n", v_level); - os_abort(); + log_fatal(ECONTENT, "Invalid parity level '%u' in the configuration file!\n", v_level); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3796,7 +3581,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' split has an invalid path\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3805,7 +3589,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' split has an invalid uuid\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3814,7 +3597,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(EINTERNAL, "Internal inconsistency: Field 'Q' split has an invalid size\n"); os_abort(); /* LCOV_EXCL_STOP */ } @@ -3823,16 +3605,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (v_size > INT64_MAX) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' split has an out-of-range size %" PRIu64 "!\n", v_size); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Invalid split file size %" PRIu64 "!\n", v_size); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } if (v_size > INT64_MAX - v_size_total) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' total parity split size is too large!\n"); - os_abort(); + log_fatal(ECONTENT, "Internal inconsistency: Total parity split size is too large!\n"); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3841,16 +3623,16 @@ static void state_read_content(struct snapraid_state* state, const char* path, S if (!has_block_size) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' missing 'blocksize' parity split information in the content file!\n"); - os_abort(); + log_fatal(ECONTENT, "Missing 'blocksize' before parity split information in the content file!\n"); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } if (v_size % state->block_size != 0) { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Field 'Q' parity split size %" PRIu64 " not aligned to block size %u!\n", v_size, state->block_size); - os_abort(); + log_fatal(ECONTENT, "Invalid parity split size %" PRIu64 " not aligned to block size %u!\n", v_size, state->block_size); + exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -3931,42 +3713,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S } else { /* LCOV_EXCL_START */ decoding_error(path, f); - log_fatal(ECONTENT, "Internal inconsistency: Invalid field command '%c'!\n", (char)c); + log_fatal(ECONTENT, "Invalid command '%c'!\n", (char)c); os_abort(); /* LCOV_EXCL_STOP */ } } - /* - * Reserve hash table bucket capacity upfront and populate containers for - * all loaded files. Deferring this until after stream parsing avoids cache - * thrashing during I/O and ensures zero dynamic resizes in tommy_hashdyn. - */ - for (uint32_t i = 0; i < mapping_max; ++i) { - struct snapraid_disk* disk = tommy_array_get(&disk_mapping, i); - tommy_size_t count = disk_file_count[i]; - tommy_node* node; - - if (count == 0) - continue; - - tommy_hashdyn_reserve(&disk->pathset, count); - tommy_hashdyn_reserve(&disk->inodeset, count); - tommy_hashdyn_reserve(&disk->stampset, count); - - node = tommy_list_head(&disk->filelist); - while (node) { - struct snapraid_file* file = node->data; - - if (file->inode != INODE_INVALID) - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file->pathset.index); - tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); - - node = node->next; - } - } - tommy_array_done(&disk_mapping); if (serror(f)) { @@ -4020,7 +3772,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* check that the stored parity size matches the loaded state */ if (blockmax != parity_allocated_size(state)) { /* LCOV_EXCL_START */ - log_fatal(EINTERNAL, "Wrong parity size %" PRIu64 "/%" PRIu64 " in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f)); + log_fatal(EINTERNAL, "Internal inconsistency: Parity size %" PRIu64 "/%" PRIu64 " in '%s' at offset %" PRIi64 "\n", blockmax, parity_allocated_size(state), path, stell(f)); if (state->opt.skip_content_check) { log_fatal(ECONTENT, "Overriding.\n"); blockmax = parity_allocated_size(state); @@ -4030,16 +3782,6 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_STOP */ } - /* - * Every invalid file or deleted run expands this window while parsing. - * Outside it all parity is valid, so no position can be unsynced. - * Count only after all records are loaded, independently of the info record. - * A synchronized content leaves the window empty and needs no sweep. - */ - if (inv_min < inv_max) { - count_unsynced = fs_count_unsynced(state, inv_min, inv_max); - } - context->blockmax = blockmax; context->count_map = mapping_max; context->count_file = count_file; @@ -4098,15 +3840,14 @@ struct state_write_context { * copied with a single memcpy(). Keep the stream pointer local while filling * each buffered group to avoid the per-block stream call and field updates. */ -__aligned_loops(64) static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); - size_t hash_size = BLOCK_HASH_SIZE; + size_t block_stride = block_sizeof(); while (count) { size_t available = (size_t)(f->end - f->pos); - block_off_t cached_count = available / hash_size; + block_off_t cached_count = available / BLOCK_HASH_SIZE; unsigned char* output; if (cached_count > count) @@ -4115,39 +3856,26 @@ static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_of if (cached_count == 0) { struct snapraid_block* block = (struct snapraid_block*)block_ptr; - if (swrite(block->hash, hash_size, f) != 0) + if (swrite(block->hash, BLOCK_HASH_SIZE, f) != 0) return -1; - block_ptr += 1 + hash_size; + block_ptr += block_stride; --count; continue; } - count -= cached_count; output = f->pos; - if (hash_size == HASH_MAX) { - while (cached_count != 0) { - struct snapraid_block* block = (struct snapraid_block*)block_ptr; + for (block_off_t i = 0; i < cached_count; ++i) { + struct snapraid_block* block = (struct snapraid_block*)block_ptr; - memcpy(output, block->hash, HASH_MAX); + hash_copy(output, block->hash); - output += HASH_MAX; - block_ptr += 1 + HASH_MAX; - --cached_count; - } - } else { - while (cached_count != 0) { - struct snapraid_block* block = (struct snapraid_block*)block_ptr; - - hash_copy(output, block->hash); - - output += hash_size; - block_ptr += 1 + hash_size; - --cached_count; - } + output += BLOCK_HASH_SIZE; + block_ptr += block_stride; } f->pos = output; + count -= cached_count; } return 0; @@ -4266,7 +3994,7 @@ static void* state_write_thread(void* arg) if (!disk) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: Unmapped disk '%s'\n", map->name); - os_abort(); + goto bail; /* LCOV_EXCL_STOP */ } @@ -4396,7 +4124,7 @@ static void* state_write_thread(void* arg) if (v_state != block_state_get(block)) break; - /* resolve the mapping only when crossing an extent boundary. */ + /* Resolve the mapping only when crossing an extent boundary. */ if (end == extent_end) { block_off_t next_count; block_off_t next_pos = fs_file2par_get_run(disk, file, end, &next_count); @@ -4427,7 +4155,7 @@ static void* state_write_thread(void* arg) default : /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency: State for block %" PRIu64 " state %u\n", v_pos, v_state); - os_abort(); + goto bail; /* LCOV_EXCL_STOP */ } @@ -5422,37 +5150,9 @@ void state_read(struct snapraid_state* state) log_tag("content_info:read_unixtime:%" PRId64 "\n", (int64_t)st.st_mtime); } log_flush(); -} -/** - * Check if all configured parity splits have persistent sizes. - */ -int state_is_write_ready(struct snapraid_state* state) -{ - unsigned l, s; - - for (l = 0; l < state->level; ++l) { - for (s = 0; s < state->parity[l].split_mac; ++s) { - if (state->parity[l].split_map[s].size == PARITY_SIZE_INVALID) - return 0; - } - } - - return 1; -} - -/** - * Require sync to initialize parity sizes before commands that can save content. - * Check before modifying timestamps or metadata, as a later write could fail - * even if the command has no blocks to process (e.g. a missing content replica). - */ -void state_require_write_ready(struct snapraid_state* state) -{ - if (!state_is_write_ready(state)) { - log_fatal(EUSER, "The array has not been synced yet, or its parity configuration has changed.\n"); - log_fatal(EUSER, "Run 'snapraid sync' before '%s'.\n", state->command); - exit(EXIT_FAILURE); - } + /* mark that we read the content file, and it passed all the checks */ + state->checked_read = 1; } struct state_verify_thread_context { @@ -5729,7 +5429,7 @@ static void state_rename_content(struct snapraid_state* state) #endif } -/** +/* * Check if a modification time nanosecond value can be represented in the * content format. * @@ -5741,7 +5441,7 @@ static int state_write_mtime_nsec_valid(int mtime_nsec) return mtime_nsec == STAT_NSEC_INVALID || (mtime_nsec >= 0 && mtime_nsec < 1000000000); } -/** +/* * Validate invariants required by the persistent content format. * * state_write() must never serialize a state that state_read() would reject. @@ -5834,7 +5534,7 @@ static void state_write_check(struct snapraid_state* state) } } -/** +/* * Emit structured mapping tags for data disks and parity splits after * content files have been durably published. */ @@ -5963,6 +5663,7 @@ void state_write(struct snapraid_state* state) log_flush(); state->need_write = 0; /* no write needed anymore */ + state->checked_read = 0; /* what we wrote is not checked in read */ state->written = 1; } diff --git a/cmdline/state.h b/cmdline/state.h index d9a93f29..993466a7 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef STATE_H -#define STATE_H +#ifndef __STATE_H +#define __STATE_H #include "elem.h" @@ -130,6 +130,7 @@ struct snapraid_state { uint64_t autosave; /**< Autosave after the specified amount of data. 0 to disable. */ int need_write; /**< If the state is changed. */ int written; /**< If the state was written at least one time */ + int checked_read; /**< If the state was read and checked. */ uint32_t block_size; /**< Block size in bytes. */ unsigned raid_mode; /**< Raid mode to use. RAID_MODE_DEFAULT or RAID_MODE_ALTERNATE. */ int file_mode; /**< File access mode. Combination of ADVISE_* flags. */ @@ -166,7 +167,7 @@ struct snapraid_state { tommy_list bucketlist; /**< Sorted list of info bucket. Derived from infoarr, but updated only state_read()/state_write() */ block_off_t bucketcount; /**< Sum of all the counts of the list */ - /* thermal */ + /* Thermal */ tommy_list thermallist; /**< List of all thermal state */ int thermal_stop_gathering; /**< If thermal data gathering is stopped */ int thermal_system_temperature; /**< System temperature. 0 if not available */ @@ -239,23 +240,6 @@ void state_probe(struct snapraid_state* state); */ void state_read(struct snapraid_state* state); -/** - * Check if all configured parity splits have persistent sizes. - */ -int state_is_write_ready(struct snapraid_state* state); - -/** - * Require initialized parity split sizes before commands that can save content. - */ -void state_require_write_ready(struct snapraid_state* state); - -/** - * Count the number of unsynced parity positions using an event sweep across extents. - * Counts positions in [blockmin, blockmax). The caller must ensure that the disk list, - * extent trees, and block states remain unchanged for the duration of the call. - */ -uint64_t fs_count_unsynced(struct snapraid_state* state, block_off_t blockmin, block_off_t blockmax); - /** * Write the new state. */ diff --git a/cmdline/str.c b/cmdline/str.c index 45dbb3c2..aac50593 100644 --- a/cmdline/str.c +++ b/cmdline/str.c @@ -493,7 +493,7 @@ static inline int is_wild_escape(char c) #endif } -/** +/* * Match with the previous pattern character provided in prev. * * An initial '**' needs to know if it was preceded by '/' to distinguish diff --git a/cmdline/str.h b/cmdline/str.h index 976ae435..a11c8a0c 100644 --- a/cmdline/str.h +++ b/cmdline/str.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2025 Andrea Mazzoleni -#ifndef STR_H -#define STR_H +#ifndef __STR_H +#define __STR_H #include "app.h" #include "str.h" @@ -97,7 +97,7 @@ char* strtrim(char* s); */ char* strlwr(char* s); -/** +/* * Find the first occurrence of 'needle' in 'haystack' only if it * appears as a separate word (space, number or string boundaries). * @@ -127,7 +127,7 @@ unsigned long long strtonll(const char* nptr, char** endptr, int base); /****************************************************************************/ /* match */ -/** +/* * Wild match function. * * If match_sub is !=0, it matches sub directory. Specifically it matches if the diff --git a/cmdline/stream.c b/cmdline/stream.c index dd978473..a7c49e33 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -691,7 +691,7 @@ int sgetble32(STREAM* f, uint32_t* value) return 0; } -ssize_t sgetbs(STREAM* f, char* str, size_t size) +int sgetbs(STREAM* f, char* str, size_t size) { uint64_t len; @@ -709,13 +709,7 @@ ssize_t sgetbs(STREAM* f, char* str, size_t size) str[len] = 0; - if (sread(f, str, len) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - return len; + return sread(f, str, len); } int swrite_uncached(const void* void_data, size_t size, STREAM* f) diff --git a/cmdline/stream.h b/cmdline/stream.h index b39b98c0..ea9d52f6 100644 --- a/cmdline/stream.h +++ b/cmdline/stream.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef STREAM_H -#define STREAM_H +#ifndef __STREAM_H +#define __STREAM_H #include "support.h" #include "util.h" @@ -329,7 +329,7 @@ int sgetble32(STREAM* f, uint32_t* value); * Read a binary string. * Return -1 on error or if the buffer is too small, or the number of chars read. */ -ssize_t sgetbs(STREAM* f, char* str, size_t size); +int sgetbs(STREAM* f, char* str, size_t size); /****************************************************************************/ /* put */ diff --git a/cmdline/support.c b/cmdline/support.c index 13983ec6..ae01598c 100644 --- a/cmdline/support.c +++ b/cmdline/support.c @@ -52,6 +52,7 @@ char* esc_buf(void) */ #if HAVE_THREAD static thread_mutex_t msg_lock; +static thread_mutex_t memory_lock; static thread_mutex_t random_lock; #endif @@ -69,6 +70,20 @@ void unlock_msg(void) #endif } +void lock_memory(void) +{ +#if HAVE_THREAD + thread_mutex_lock(&memory_lock); +#endif +} + +void unlock_memory(void) +{ +#if HAVE_THREAD + thread_mutex_unlock(&memory_lock); +#endif +} + void lock_random(void) { #if HAVE_THREAD @@ -88,6 +103,7 @@ void lock_init(void) #if HAVE_THREAD /* initialize the locks as first operation as log_fatal depends on them */ thread_mutex_init(&msg_lock); + thread_mutex_init(&memory_lock); thread_mutex_init(&random_lock); thread_key_create(&esc_key, esc_pool_free); #endif @@ -103,6 +119,7 @@ void lock_done(void) } thread_key_delete(esc_key); thread_mutex_destroy(&msg_lock); + thread_mutex_destroy(&memory_lock); thread_mutex_destroy(&random_lock); #endif } @@ -133,7 +150,7 @@ unsigned char random_u8(void) return random_u64() & 0xFF; } -/** +/* * SplitMix64 * * - Zero-safe: Any 64-bit seed is valid (even 0). @@ -293,7 +310,7 @@ unsigned log_hardware_errors(void) return count; } -/** +/* * Note that in the following functions we always flush both * stdout and stderr, because we want to ensure that they mixes * well when redirected to files @@ -395,7 +412,7 @@ void vlog_fatal(int err, const char* format, va_list ap, const char* post) log_failed = 1; } - /* a failed tagged write must retain stderr as the only usable fatal diagnostic. */ + /* A failed tagged write must retain stderr as the only usable fatal diagnostic. */ if (!stdlog || log_failed || msg_fatal_stderr) { va_list ap_copy2; va_copy(ap_copy2, ap); @@ -1108,7 +1125,7 @@ void pathimport(char* dst, size_t size, const char* src) pathcpy(dst, size, src); #ifdef _WIN32 - /* convert the Windows dir separator '\' to C '/' */ + /* Convert the Windows dir separator '\' to C '/' */ while (*dst) { if (*dst == '\\') *dst = '/'; @@ -1122,7 +1139,7 @@ void pathexport(char* dst, size_t size, const char* src) pathcpy(dst, size, src); #ifdef _WIN32 - /* convert C '/' to Windows dir separator '\' */ + /* Convert C '/' to Windows dir separator '\' */ while (*dst) { if (*dst == '/') *dst = '\\'; @@ -1928,7 +1945,7 @@ int is_sep(char c) || c == (char)0xa0; /* french */ } -/** +/* * snumber() * * Matches a literal prefix (the second argument), then parses the following diff --git a/cmdline/support.h b/cmdline/support.h index ec05ed9e..995901f8 100644 --- a/cmdline/support.h +++ b/cmdline/support.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef SUPPORT_H -#define SUPPORT_H +#ifndef __SUPPORT_H +#define __SUPPORT_H #include "app.h" #include "memory.h" @@ -36,6 +36,12 @@ void lock_done(void); void lock_msg(void); void unlock_msg(void); +/** + * Lock used for memory counter. + */ +void lock_memory(void); +void unlock_memory(void); + /****************************************************************************/ /* random */ diff --git a/cmdline/sync.c b/cmdline/sync.c index 8d8770aa..c9b505c3 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -11,7 +11,7 @@ #include "io.h" #include "raid/raid.h" -/** +/* * SnapRAID sync model * * Sync brings the stored parity and content metadata in line with the data @@ -1921,7 +1921,7 @@ bail_abort: return 0; } -/** +/* * Make a forced full parity rebuild persistent. * * REBUILD means that the current data and hash are known, but physical parity @@ -1959,7 +1959,7 @@ static void state_sync_mark_rebuild(struct snapraid_state* state) state->need_write = 1; } -/** +/* * Initialize persistent parity split sizes resolved while opening parity. * * parity_create() resolves PARITY_SIZE_INVALID in the parity handles using @@ -2208,7 +2208,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t /* after resizing parity files, refresh again the free info */ state_refresh(state); - /* + /** * Save the new state before the sync but after the hashing phase * * This allows to recover after an aborted sync, and at the same time diff --git a/cmdline/thermal.c b/cmdline/thermal.c index ff0c05d7..821d9675 100644 --- a/cmdline/thermal.c +++ b/cmdline/thermal.c @@ -30,7 +30,7 @@ void thermal_free(void* void_thermal) free(thermal); } -/** +/* * Fit exponential heating model to data using least squares */ struct snapraid_thermal_params fit_thermal_model(const struct snapraid_thermal_point* points, int n_points, double t_ambient) diff --git a/cmdline/thermal.h b/cmdline/thermal.h index 5ae21587..5e23ddb3 100644 --- a/cmdline/thermal.h +++ b/cmdline/thermal.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2025 Andrea Mazzoleni -#ifndef THERMAL_H -#define THERMAL_H +#ifndef __THERMAL_H +#define __THERMAL_H #include "elem.h" diff --git a/cmdline/touch.c b/cmdline/touch.c index dff3d3c0..192dcdd4 100644 --- a/cmdline/touch.c +++ b/cmdline/touch.c @@ -90,14 +90,6 @@ int state_touch(struct snapraid_state* state) } #endif if (f == -1) { - /* - * Ignore missing files because they may have been deleted - * or moved since the last sync. Also ignore symbolic links - * which fail with ELOOP due to O_NOFOLLOW. - */ - if (errno == ENOENT || errno == ENOTDIR || errno == ELOOP) - continue; - /* LCOV_EXCL_START */ log_error(errno, "Error opening file '%s'. %s.\n", path, strerror(errno)); ++error; @@ -120,12 +112,11 @@ int state_touch(struct snapraid_state* state) } /* - * Touch only regular files that are still unchanged compared to + * Touch only files that are still unchanged compared to * the content file. Otherwise changing the timestamp here * could hide a modification from the following scan. */ - if (!S_ISREG(st.st_mode) - || st.st_size != file->size + if (st.st_size != file->size || st.st_mtime != file->mtime_sec || STAT_NSEC(&st) != file->mtime_nsec ) { diff --git a/cmdline/unixapp.c b/cmdline/unixapp.c index 0e23e0de..20e52be8 100644 --- a/cmdline/unixapp.c +++ b/cmdline/unixapp.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifndef __MINGW32__ /* only for Unix */ +#ifndef __MINGW32__ /* Only for Unix */ #include "app.h" #include "support.h" @@ -64,7 +64,7 @@ static inline unsigned statfs_type(const struct statfs* sfs) #if HAVE_LINUX_DEVICE static const char* const bcachefs_paths[] = { #ifdef BCACHEFS_PATH - /* path configured at build time (e.g. on NixOS). */ + /* Path configured at build time (e.g. on NixOS). */ BCACHEFS_PATH, #else "/usr/sbin/bcachefs", @@ -96,7 +96,7 @@ static const char* find_bcachefs(void) #if HAVE_LINUX_DEVICE static const char* const zfs_paths[] = { #ifdef ZFS_PATH - /* path configured at build time (e.g. on NixOS). */ + /* Path configured at build time (e.g. on NixOS). */ ZFS_PATH, #else "/usr/sbin/zfs", @@ -125,7 +125,7 @@ static const char* find_zfs(void) static const char* const zpool_paths[] = { #ifdef ZPOOL_PATH - /* path configured at build time (e.g. on NixOS). */ + /* Path configured at build time (e.g. on NixOS). */ ZPOOL_PATH, #else "/usr/sbin/zpool", @@ -157,7 +157,7 @@ static const char* find_zpool(void) static const char* smartctl_paths[] = { #ifdef SMARTCTL_PATH - /* path configured at build time (e.g. on NixOS). */ + /* Path configured at build time (e.g. on NixOS). */ SMARTCTL_PATH, #else /* Linux & BSD */ @@ -228,7 +228,7 @@ static ssize_t sysread(const char* path, char* buf, size_t buf_size) #endif #if HAVE_LINUX_DEVICE -/** +/* * sysattr_vpd_pg80 — parse raw binary VPD page 0x80 (Unit Serial Number) * from a sysfs vpd_pg80 file. */ @@ -291,7 +291,7 @@ static int sysattr_vpd_pg80(const char* path, char* dst, size_t dst_size) } #endif -/** +/* * sysread_vpd_pg83 — parse raw binary VPD page 0x83 (Device Identification) * from a sysfs vpd_pg83 file and extract the first NAA (Network Address * Authority) designator for the Logical Unit (ASSOCIATION == 0x00). @@ -418,7 +418,7 @@ static int sysattr_vpd_pg83(const char* path, char* dst, size_t dst_size) switch (naa) { case 0x1 : /* IEEE Extended */ - case 0x2 : /* locally assigned */ + case 0x2 : /* Locally Assigned */ case 0x3 : /* IEEE Registered */ case 0x5 : /* IEEE Registered */ expected = 8; @@ -503,7 +503,7 @@ static long long syslong(const char* path) return v; } -/** +/* * Parse Linux cpulist format (e.g. "0-3,5,8-11") into an array of CPU IDs. * * Returns the total number of CPUs in the list. Stores up to max_cpus entries. @@ -553,7 +553,7 @@ static unsigned parse_cpulist(const char* str, unsigned* cpus, unsigned max_cpus return count; } -/** +/* * Read online CPU IDs from /sys/devices/system/cpu/online. * * Falls back to consecutive 0..N-1 IDs if sysfs is unavailable. @@ -584,7 +584,7 @@ static unsigned read_online_cpus(unsigned* cpus, unsigned max_cpus) return count; } -/** +/* * Returns 1 if the specified cpu is the lowest-numbered logical CPU in its * physical core's SMT sibling set. */ @@ -2737,7 +2737,7 @@ static int fssnapshot_stat_fs(const struct fssnapshot_struct* fss, const char* n } #endif -/** +/* * Native ZFS snapshots use a SnapRAID-specific prefix to avoid * deleting or renaming unrelated user snapshots with generic * names such as @scan, @pending, or @stable. @@ -2772,7 +2772,7 @@ static int fssnapshot_stat_zfs(const struct fssnapshot_struct* fss, const char* return -1; } - /* + /** * When verifying if a ZFS snapshot has been successfully deleted, a direct * path lookup (stat, access, open) is unreliable. A directory listing * (readdir) of the .zfs/snapshot directory must be used instead. @@ -4375,16 +4375,6 @@ int devquery(tommy_list* high, tommy_list* low) struct dev_struct* dev = j->data; uint64_t access_stat; - /* - * Major 0 is an anonymous superblock (NFS, CIFS, FUSE, tmpfs, overlay, ...) - * not backed by a block device. Btrfs/Bcachefs/ZFS are already dereferenced - * to their member devices, so what remains has no physical disk to manage. - */ - if (major(dev->device) == 0) { - log_tag("device:%s:skip:%u:%u: no block device\n", esc_tag(devinfo->name), major(dev->device), minor(dev->device)); - continue; - } - /* retrieve access stat for the high level device */ if (devstat(dev->device, &access_stat) == 0) { /* cumulate access stat in the first split */ @@ -4430,9 +4420,6 @@ int devquery(tommy_list* high, tommy_list* low) #else (void)high; (void)low; - - /* device operations are unsupported in this platform */ - failed = 1; #endif return failed ? -1 : degraded ? 1 : 0; @@ -4460,7 +4447,6 @@ int devmap(void) { #if HAVE_LINUX_DEVICE char path[PATH_MAX]; - char boot_id[64]; DIR* d = opendir("/sys/block"); if (!d) { /* LCOV_EXCL_START */ @@ -4468,22 +4454,6 @@ int devmap(void) /* LCOV_EXCL_STOP */ } - /* optional boot identity: failure must not affect hardware mappings */ - if (sysattr("/proc/sys/kernel/random/boot_id", boot_id, sizeof(boot_id)) != 0 || strlen(boot_id) != 36) { - boot_id[0] = 0; - } else { - for (unsigned i = 0; i < 36; ++i) { - if (i == 8 || i == 13 || i == 18 || i == 23) { - if (boot_id[i] == '-') - continue; - } else if (hexval(boot_id[i]) >= 0) { - continue; - } - boot_id[0] = 0; - break; - } - } - struct dirent* dd; while (1) { char buf[256]; @@ -4547,20 +4517,6 @@ int devmap(void) if (sysattr(path, buf, sizeof(buf)) == 0 && buf[0] != 0) { log_tag("map:/dev/%s:%s\n", dd->d_name, esc_tag(buf)); } - - /* - * This identifies the kernel device instance within one boot, not - * persistent hardware across reconnections. Missing attributes are optional. - */ - if (boot_id[0] != 0) { - pathprint(path, sizeof(path), "/sys/block/%s/diskseq", dd->d_name); - if (sysattr(path, buf, sizeof(buf)) == 0) { - char* end; - unsigned long long diskseq = strtonll(buf, &end, 10); - if (buf[0] >= '0' && buf[0] <= '9' && diskseq != 0 && end != buf && *end == 0) - log_tag("map:/dev/%s:linux-diskseq.%s.%llu\n", esc_tag(dd->d_name), esc_tag(boot_id), diskseq); - } - } } closedir(d); @@ -4574,7 +4530,7 @@ int devmap(void) #if HAVE_LINUX_DEVICE -/* list of possible ambient temperature labels */ +/* List of possible ambient temperature labels */ const char* AMBIENT_LABEL[] = { "systin", "auxtin", @@ -5020,7 +4976,7 @@ void machineinfo(struct machineinfo_struct* info) } } - /* fallback if sysfs cpufreq is unavailable (e.g. Cloud VM) */ + /* Fallback if sysfs cpufreq is unavailable (e.g. Cloud VM) */ if (info->cpu_clock[0] == 0) { #ifdef CONFIG_X86 raid_cpuid(0, 0, reg); diff --git a/cmdline/util.c b/cmdline/util.c index 2cc659b6..233656f8 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -7,7 +7,6 @@ #include "util.h" #include "raid/cpu.h" #include "raid/memory.h" -#include "tommyds/tommyhash.h" /****************************************************************************/ /* memory */ @@ -434,7 +433,7 @@ static void crc32c_skip_init(uint32_t table[8][16], size_t size) unsigned i; unsigned j; - /* collapse each fixed-size shift into eight nibble table lookups. */ + /* Collapse each fixed-size shift into eight nibble table lookups. */ for (i = 0; i < 8; ++i) { for (j = 0; j < 16; ++j) table[i][j] = crc32c_shift((uint32_t)j << (4 * i), size); @@ -454,7 +453,6 @@ uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, size_t size) } #if CONFIG_X86 -__aligned_loops(64) uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) { crc ^= CRC_IV; @@ -468,7 +466,6 @@ uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) #endif #if CONFIG_ARM_CRC -__aligned_loops(64) uint32_t crc32c_arm64(uint32_t crc, const unsigned char* ptr, size_t size) { crc ^= CRC_IV; @@ -515,7 +512,7 @@ void crc32c_init(void) /****************************************************************************/ /* byte operations */ -/** +/* * Rotate left. * In x86/x64/ARM they are optimized with a single assembler instruction. */ @@ -529,7 +526,7 @@ static inline uint64_t util_rotl64(uint64_t x, unsigned r) return (x << (r & 63)) | (x >> ((-r) & 63)); } -/** +/* * Rotate right. * In x86/x64/ARM they are optimized with a single assembler instruction. */ @@ -611,6 +608,112 @@ static inline void util_write64(void* ptr, uint64_t v) memcpy(ptr, &v, sizeof(v)); } +/* + * 64*64 -> 128bit multiply function. + * + * @param A Address of 64-bit number. + * @param B Address of 64-bit number. + * + * Calculates 128-bit C = *A * *B. + * + * Overwrites A contents with C's low 64 bits. + * Overwrites B contents with C's high 64 bits. + */ +#if defined(__x86_64__) || defined(_M_X64) +/* + * This is intentionally placed before the __SIZEOF_INT128__ implementation. + * + * This implementation happens to generate more efficient code. + * While this likely depends only on the compiler, as long as it remains faster, + * we will keep it in this order. + */ +static inline void util_mum(uint64_t* A, uint64_t* B) +{ + uint64_t a = *A; + uint64_t b = *B; + uint64_t low, high; + + __asm__ ( + "mulq %3" + : "=a" (low), "=d" (high) + : "a" (a), "rm" (b) + : "cc" + ); + + *A = low; + *B = high; +} +#elif defined(__SIZEOF_INT128__) +static inline void util_mum(uint64_t* A, uint64_t* B) +{ + __uint128_t product = ((__uint128_t)*A) * ((__uint128_t)*B); + *A = (uint64_t)product; + *B = (uint64_t)(product >> 64); +} +#elif defined(__aarch64__) || defined(_M_ARM64) +/* + * This is intentionally placed after the __SIZEOF_INT128__ implementation. + * + * Although the code looks correct, there is a report of this + * implementation failing the self-test on macOS (Darwin 25.4.0, arm64) + * with Apple clang 21.0.0 (clang-2100.0.123.102). + * + * See: https://github.com/amadvance/snapraid/issues/43 + * + * Note that the implementation is the same of OpenSSL: + * + * #elif defined(__aarch64__) && defined(SIXTY_FOUR_BIT_LONG) + * #if defined(__GNUC__) && __GNUC__ >= 2 + * #define BN_UMULT_HIGH(a, b) ({ \ + * register BN_ULONG ret; \ + * asm ("umulh %0,%1,%2" \ + * : "=r"(ret) \ + * : "r"(a), "r"(b)); \ + * ret; }) + * #endif + * #endif + * + * See: https://github.com/openssl/openssl/blob/0463cbf185680e66a3a8960f10cd33001377d28e/crypto/bn/bn_local.h#L439 + */ +static inline void util_mum(uint64_t* A, uint64_t* B) +{ + uint64_t a = *A; + uint64_t b = *B; + uint64_t low = a * b; + uint64_t high; + __asm__ ( + "umulh %0, %1, %2" + : "=r" (high) + : "r" (a), "r" (b) + ); + *A = low; + *B = high; +} +#else +static inline void util_mum(uint64_t* A, uint64_t* B) +{ + uint64_t a = *A; + uint64_t b = *B; + + uint64_t a_lo = (uint32_t)a; + uint64_t a_hi = a >> 32; + uint64_t b_lo = (uint32_t)b; + uint64_t b_hi = b >> 32; + + uint64_t p0 = a_lo * b_lo; + uint64_t p1 = a_lo * b_hi; + uint64_t p2 = a_hi * b_lo; + uint64_t p3 = a_hi * b_hi; + + uint64_t middle = p1 + (p0 >> 32) + (uint32_t)p2; + uint64_t low = (middle << 32) | (uint32_t)p0; + uint64_t high = p3 + (middle >> 32) + (p2 >> 32); + + *A = low; + *B = high; +} +#endif + int util_selftest(void) { /* rotate left 32-bit */ @@ -643,33 +746,33 @@ int util_selftest(void) if (util_rotr64(0x123456789ABCDEF0ULL, 0) != 0x123456789ABCDEF0ULL) return -1; if (util_rotr64(0x123456789ABCDEF0ULL, 64) != 0x123456789ABCDEF0ULL) return -1; - /* tommy_mum tests */ + /* util_mum tests */ uint64_t a, b; /* small values */ a = 3; b = 4; - tommy_mum(&a, &b); + util_mum(&a, &b); if (a != 12 || b != 0) return -1; /* 1 * anything */ a = 0xFFFFFFFFFFFFFFFFULL; b = 1; - tommy_mum(&a, &b); + util_mum(&a, &b); if (a != 0xFFFFFFFFFFFFFFFFULL || b != 0) return -1; /* (2^32) * (2^32) = 2^64 */ a = 1ULL << 32; b = 1ULL << 32; - tommy_mum(&a, &b); + util_mum(&a, &b); if (a != 0 || b != 1) return -1; /* max uint64 * max uint64 */ a = 0xFFFFFFFFFFFFFFFFULL; b = 0xFFFFFFFFFFFFFFFFULL; - tommy_mum(&a, &b); + util_mum(&a, &b); if (a != 1 || b != 0xFFFFFFFFFFFFFFFEULL) return -1; /* larger random-ish test */ a = 0x0123456789ABCDEFULL; b = 0xFEDCBA9876543210ULL; - tommy_mum(&a, &b); + util_mum(&a, &b); if (a != 0x2236D88FE5618CF0ULL || b != 0x121FA00AD77D742ULL) return -1; return 0; @@ -723,7 +826,7 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* default : /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Internal inconsistency in hash function %u\n", kind); - os_abort(); + exit(EXIT_FAILURE); break; /* LCOV_EXCL_STOP */ } diff --git a/cmdline/util.h b/cmdline/util.h index b9b4f25e..f65dce31 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (C) 2011 Andrea Mazzoleni -#ifndef UTIL_H -#define UTIL_H +#ifndef __UTIL_H +#define __UTIL_H #include "tommyds/tommytypes.h" @@ -110,11 +110,6 @@ uint32_t crc32c_shift(uint32_t crc, size_t size); */ static inline uint32_t crc32c_gen_plain(uint32_t crc, const unsigned char* ptr, size_t size) { -#if defined(__GNUC__) && !defined(__clang__) -#pragma GCC unroll 1 -#elif defined(__clang__) -#pragma unroll 1 -#endif while (size >= 4) { crc ^= ptr[0] | (uint32_t)ptr[1] << 8 | (uint32_t)ptr[2] << 16 | (uint32_t)ptr[3] << 24; crc = CRC32C_3[crc & 0xff] ^ CRC32C_2[(crc >> 8) & 0xff] ^ CRC32C_1[(crc >> 16) & 0xff] ^ CRC32C_0[crc >> 24]; @@ -148,7 +143,6 @@ static __always_inline uint32_t crc32c_x86_64_skip(uint32_t crc, const uint32_t ^ table[7][crc >> 28]; } -__aligned_loops(64) static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned char* ptr, size_t lane_size, const uint32_t table[8][16]) { const unsigned char* ptr0 = ptr; @@ -181,7 +175,6 @@ static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned } #endif -__aligned_loops(64) static __always_inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, size_t size) { #ifdef CONFIG_X86_64 diff --git a/configure.ac b/configure.ac index b80ad099..dbc75081 100644 --- a/configure.ac +++ b/configure.ac @@ -71,7 +71,7 @@ AC_HEADER_ASSERT AC_HEADER_SYS_WAIT AC_CHECK_HEADERS([dirent.h stdint.h inttypes.h unistd.h math.h execinfo.h strings.h getopt.h syslog.h grp.h pwd.h io.h alloca.h]) AC_CHECK_HEADERS([sys/file.h sys/sysctl.h sys/ioctl.h sys/time.h sys/types.h sys/mkdev.h sys/sysmacros.h sys/stat.h sys/prctl.h sys/sysinfo.h sys/utsname.h]) -AC_CHECK_HEADERS([linux/fs.h linux/btrfs.h linux/fiemap.h]) +AC_CHECK_HEADERS([linux/fs.h linux/btrfs.h linux/fiemap.h mach/mach_time.h]) # Check for the close_range(...,CLOSE_RANGE_CLOEXEC) AC_CHECK_HEADERS([linux/close_range.h]) @@ -126,6 +126,7 @@ AC_CHECK_FUNCS([fsync posix_fadvise sync_file_range syncfs]) AC_CHECK_FUNCS([getc_unlocked ferror_unlocked]) AC_CHECK_FUNCS([futimes futimens futimesat localtime_r lutimes utimensat]) AC_CHECK_FUNCS([fstatat flock renameat statx]) +AC_CHECK_FUNCS([mach_absolute_time]) AC_CHECK_FUNCS([backtrace backtrace_symbols]) AC_SEARCH_LIBS([clock_gettime], [rt]) AC_CHECK_FUNCS([clock_gettime]) diff --git a/doc/snapraid-de.1 b/doc/snapraid-de.1 index a64fdd51..f2714b9f 100644 --- a/doc/snapraid-de.1 +++ b/doc/snapraid-de.1 @@ -2075,37 +2075,6 @@ Wenn ein \`sync\` jedoch unterbrochen wird, vermeiden Sie es, Dateien zu die aus dem vom unterbrochenen Sync verwendeten Snapshot berechnet wurden. Ein späterer Sync könnte diesen Snapshot ersetzen, wodurch Zwischenversionen von Dateien für die Wiederherstellung nicht mehr verfügbar sind. -.SS Externe Sicherungen mit Dateisystem\-Snapshots (External backups with filesystem snapshots) -Wenn Dateisystem\-Snapshots aktiviert sind, greift SnapRAID auf Snapshots -an der Wurzel jedes Daten\-Subvolumes, Volumes oder Datasets zu oder erstellt -diese dort. -.PP -Ein einfacher Weg, um jegliche Beeinträchtigung durch externe Tools zu -vermeiden, besteht darin, die \`data\`\-Konfiguration auf ein reguläres -Unterverzeichnis innerhalb des Subvolumes oder Volumes verweisen zu lassen -(beispielsweise \`/mnt/subvol/data\` statt \`/mnt/subvol\`). SnapRAID erstellt -das Snapshot\-Verzeichnis immer an der Wurzel des Subvolumes (beispielsweise -\`/mnt/subvol/.snapraid/\`), wodurch das Datenunterverzeichnis sauber bleibt -und Snapshots außerhalb und unsichtbar für Sicherungswerkzeuge bleiben, die -darin arbeiten. -.PP -Wenn das konfigurierte \`data\`\-Verzeichnis hingegen selbst die Wurzel des -Subvolumes oder Volumes ist, erstellt SnapRAID ein \`.snapraid/\`\-Verzeichnis -direkt darin (auf Btrfs, Bcachefs und NTFS). Während SnapRAID dieses -Verzeichnis bei seinen eigenen Scans automatisch ausschließt, werden -externe Sicherungs\- oder Synchronisationswerkzeuge darauf stoßen. Wenn es -nicht ausgeschlossen wird, könnten sie in die Snapshots oder Verknüpfungen -eindringen und den gesamten eingefrorenen Datenbaum kopieren, was Daten -dupliziert und den Sicherungsspeicherplatz erschöpfen kann. Konfigurieren -Sie externe Sicherungswerkzeuge immer so, dass sie das Verzeichnis -\`.snapraid/\` ausschließen. -.PP -Bei ZFS werden Snapshots über das standardmäßige virtuelle Verzeichnis -\`.zfs/snapshot/\` anstelle von \`.snapraid/\` bereitgestellt. Standardmäßig -(\`snapdir=hidden\`) blendet ZFS dieses Verzeichnis in Verzeichnislisten aus, -sodass externe Sicherungswerkzeuge es nicht durchlaufen. Wenn Sie -\`snapdir=visible\` aktiviert haben, stellen Sie sicher, dass Ihre -Sicherungswerkzeuge das Verzeichnis \`.zfs/\` ausschließen. .SH MUSTER (PATTERN) Muster bieten eine flexible Möglichkeit, Dateien für die Einbeziehung oder den Ausschluss zu filtern. Durch die Verwendung von Globbing\-Zeichen können Sie Regeln definieren, diff --git a/doc/snapraid-de.d b/doc/snapraid-de.d index 3371c773..70c33437 100644 --- a/doc/snapraid-de.d +++ b/doc/snapraid-de.d @@ -1669,38 +1669,6 @@ Lebenszyklus von Snapshots (Snapshots Lifecycle) Ein späterer Sync könnte diesen Snapshot ersetzen, wodurch Zwischenversionen von Dateien für die Wiederherstellung nicht mehr verfügbar sind. - Externe Sicherungen mit Dateisystem-Snapshots (External backups with filesystem snapshots) - Wenn Dateisystem-Snapshots aktiviert sind, greift SnapRAID auf Snapshots - an der Wurzel jedes Daten-Subvolumes, Volumes oder Datasets zu oder erstellt - diese dort. - - Ein einfacher Weg, um jegliche Beeinträchtigung durch externe Tools zu - vermeiden, besteht darin, die `data`-Konfiguration auf ein reguläres - Unterverzeichnis innerhalb des Subvolumes oder Volumes verweisen zu lassen - (beispielsweise `/mnt/subvol/data` statt `/mnt/subvol`). SnapRAID erstellt - das Snapshot-Verzeichnis immer an der Wurzel des Subvolumes (beispielsweise - `/mnt/subvol/.snapraid/`), wodurch das Datenunterverzeichnis sauber bleibt - und Snapshots außerhalb und unsichtbar für Sicherungswerkzeuge bleiben, die - darin arbeiten. - - Wenn das konfigurierte `data`-Verzeichnis hingegen selbst die Wurzel des - Subvolumes oder Volumes ist, erstellt SnapRAID ein `.snapraid/`-Verzeichnis - direkt darin (auf Btrfs, Bcachefs und NTFS). Während SnapRAID dieses - Verzeichnis bei seinen eigenen Scans automatisch ausschließt, werden - externe Sicherungs- oder Synchronisationswerkzeuge darauf stoßen. Wenn es - nicht ausgeschlossen wird, könnten sie in die Snapshots oder Verknüpfungen - eindringen und den gesamten eingefrorenen Datenbaum kopieren, was Daten - dupliziert und den Sicherungsspeicherplatz erschöpfen kann. Konfigurieren - Sie externe Sicherungswerkzeuge immer so, dass sie das Verzeichnis - `.snapraid/` ausschließen. - - Bei ZFS werden Snapshots über das standardmäßige virtuelle Verzeichnis - `.zfs/snapshot/` anstelle von `.snapraid/` bereitgestellt. Standardmäßig - (`snapdir=hidden`) blendet ZFS dieses Verzeichnis in Verzeichnislisten aus, - sodass externe Sicherungswerkzeuge es nicht durchlaufen. Wenn Sie - `snapdir=visible` aktiviert haben, stellen Sie sicher, dass Ihre - Sicherungswerkzeuge das Verzeichnis `.zfs/` ausschließen. - Muster (Pattern) Muster bieten eine flexible Möglichkeit, Dateien für die Einbeziehung oder den Ausschluss zu filtern. Durch die Verwendung von Globbing-Zeichen können Sie Regeln definieren, diff --git a/doc/snapraid-de.txt b/doc/snapraid-de.txt index 75776832..4793db60 100644 --- a/doc/snapraid-de.txt +++ b/doc/snapraid-de.txt @@ -1779,40 +1779,6 @@ die aus dem vom unterbrochenen Sync verwendeten Snapshot berechnet wurden. Ein späterer Sync könnte diesen Snapshot ersetzen, wodurch Zwischenversionen von Dateien für die Wiederherstellung nicht mehr verfügbar sind. -9.1 Externe Sicherungen mit Dateisystem-Snapshots (External backups with filesystem snapshots) ----------------------------------------------------------------------------------------------- - -Wenn Dateisystem-Snapshots aktiviert sind, greift SnapRAID auf Snapshots -an der Wurzel jedes Daten-Subvolumes, Volumes oder Datasets zu oder erstellt -diese dort. - -Ein einfacher Weg, um jegliche Beeinträchtigung durch externe Tools zu -vermeiden, besteht darin, die `data`-Konfiguration auf ein reguläres -Unterverzeichnis innerhalb des Subvolumes oder Volumes verweisen zu lassen -(beispielsweise `/mnt/subvol/data` statt `/mnt/subvol`). SnapRAID erstellt -das Snapshot-Verzeichnis immer an der Wurzel des Subvolumes (beispielsweise -`/mnt/subvol/.snapraid/`), wodurch das Datenunterverzeichnis sauber bleibt -und Snapshots außerhalb und unsichtbar für Sicherungswerkzeuge bleiben, die -darin arbeiten. - -Wenn das konfigurierte `data`-Verzeichnis hingegen selbst die Wurzel des -Subvolumes oder Volumes ist, erstellt SnapRAID ein `.snapraid/`-Verzeichnis -direkt darin (auf Btrfs, Bcachefs und NTFS). Während SnapRAID dieses -Verzeichnis bei seinen eigenen Scans automatisch ausschließt, werden -externe Sicherungs- oder Synchronisationswerkzeuge darauf stoßen. Wenn es -nicht ausgeschlossen wird, könnten sie in die Snapshots oder Verknüpfungen -eindringen und den gesamten eingefrorenen Datenbaum kopieren, was Daten -dupliziert und den Sicherungsspeicherplatz erschöpfen kann. Konfigurieren -Sie externe Sicherungswerkzeuge immer so, dass sie das Verzeichnis -`.snapraid/` ausschließen. - -Bei ZFS werden Snapshots über das standardmäßige virtuelle Verzeichnis -`.zfs/snapshot/` anstelle von `.snapraid/` bereitgestellt. Standardmäßig -(`snapdir=hidden`) blendet ZFS dieses Verzeichnis in Verzeichnislisten aus, -sodass externe Sicherungswerkzeuge es nicht durchlaufen. Wenn Sie -`snapdir=visible` aktiviert haben, stellen Sie sicher, dass Ihre -Sicherungswerkzeuge das Verzeichnis `.zfs/` ausschließen. - 10 MUSTER (PATTERN) =================== diff --git a/doc/snapraid-es.1 b/doc/snapraid-es.1 index 6d8294a0..ff245ca7 100644 --- a/doc/snapraid-es.1 +++ b/doc/snapraid-es.1 @@ -2034,36 +2034,6 @@ con éxito. La paridad aún puede contener bloques calculados a partir de la instantánea utilizada por el sync interrumpido. Un sync posterior puede reemplazar esa instantánea, haciendo que las versiones intermedias de los archivos no estén disponibles para la recuperación. -.SS Copias de seguridad externas con instantáneas del sistema de archivos (External backups with filesystem snapshots) -Cuando las instantáneas del sistema de archivos están habilitadas, SnapRAID -accede o crea instantáneas en la raíz de cada subvolumen, volumen o conjunto -de datos (dataset). -.PP -Una forma sencilla de evitar interferencias con herramientas externas es -apuntar la configuración \`data\` a un subdirectorio normal dentro del -subvolumen o volumen (por ejemplo, \`/mnt/subvol/data\` en lugar de -\`/mnt/subvol\`). SnapRAID siempre crea el directorio de instantáneas en la -raíz del subvolumen (por ejemplo, \`/mnt/subvol/.snapraid/\`), dejando limpio -el subdirectorio de datos para que las instantáneas permanezcan fuera e -invisibles para las herramientas de copia de seguridad que operan dentro -de él. -.PP -De lo contrario, si el directorio \`data\` configurado es en sí mismo la raíz -del subvolumen o volumen, SnapRAID crea un directorio \`.snapraid/\` -directamente dentro de él (en Btrfs, Bcachefs y NTFS). Aunque SnapRAID -excluye automáticamente este directorio de su propio escaneo, las -herramientas externas de copia de seguridad o sincronización de archivos lo -encontrarán. Si no se excluye, pueden acceder a las instantáneas o enlaces -de directorio y copiar todo el árbol de datos congelado, duplicando datos y -agotando el almacenamiento de respaldo. Configure siempre las herramientas -de copia de seguridad externas para excluir el directorio \`.snapraid/\`. -.PP -En ZFS, las instantáneas se exponen a través del directorio virtual estándar -\`.zfs/snapshot/\` en lugar de \`.snapraid/\`. De forma predeterminada -(\`snapdir=hidden\`), ZFS oculta este directorio de los listados, por lo que -las herramientas externas de copia de seguridad no lo recorrerán. Si ha -habilitado \`snapdir=visible\`, asegúrese de que sus herramientas de copia -de seguridad excluyan el directorio \`.zfs/\`. .SH PATRONES (PATTERN) Los patrones proporcionan una forma flexible de filtrar archivos para su inclusión o exclusión. Al usar caracteres de globbing, puede definir reglas que coincidan con diff --git a/doc/snapraid-es.d b/doc/snapraid-es.d index beb131fd..4eb03b86 100644 --- a/doc/snapraid-es.d +++ b/doc/snapraid-es.d @@ -1639,37 +1639,6 @@ Ciclo de vida de las instantáneas (Snapshots Lifecycle) reemplazar esa instantánea, haciendo que las versiones intermedias de los archivos no estén disponibles para la recuperación. - Copias de seguridad externas con instantáneas del sistema de archivos (External backups with filesystem snapshots) - Cuando las instantáneas del sistema de archivos están habilitadas, SnapRAID - accede o crea instantáneas en la raíz de cada subvolumen, volumen o conjunto - de datos (dataset). - - Una forma sencilla de evitar interferencias con herramientas externas es - apuntar la configuración `data` a un subdirectorio normal dentro del - subvolumen o volumen (por ejemplo, `/mnt/subvol/data` en lugar de - `/mnt/subvol`). SnapRAID siempre crea el directorio de instantáneas en la - raíz del subvolumen (por ejemplo, `/mnt/subvol/.snapraid/`), dejando limpio - el subdirectorio de datos para que las instantáneas permanezcan fuera e - invisibles para las herramientas de copia de seguridad que operan dentro - de él. - - De lo contrario, si el directorio `data` configurado es en sí mismo la raíz - del subvolumen o volumen, SnapRAID crea un directorio `.snapraid/` - directamente dentro de él (en Btrfs, Bcachefs y NTFS). Aunque SnapRAID - excluye automáticamente este directorio de su propio escaneo, las - herramientas externas de copia de seguridad o sincronización de archivos lo - encontrarán. Si no se excluye, pueden acceder a las instantáneas o enlaces - de directorio y copiar todo el árbol de datos congelado, duplicando datos y - agotando el almacenamiento de respaldo. Configure siempre las herramientas - de copia de seguridad externas para excluir el directorio `.snapraid/`. - - En ZFS, las instantáneas se exponen a través del directorio virtual estándar - `.zfs/snapshot/` en lugar de `.snapraid/`. De forma predeterminada - (`snapdir=hidden`), ZFS oculta este directorio de los listados, por lo que - las herramientas externas de copia de seguridad no lo recorrerán. Si ha - habilitado `snapdir=visible`, asegúrese de que sus herramientas de copia - de seguridad excluyan el directorio `.zfs/`. - Patrones (Pattern) Los patrones proporcionan una forma flexible de filtrar archivos para su inclusión o exclusión. Al usar caracteres de globbing, puede definir reglas que coincidan con diff --git a/doc/snapraid-es.txt b/doc/snapraid-es.txt index ecfd677c..b4e08684 100644 --- a/doc/snapraid-es.txt +++ b/doc/snapraid-es.txt @@ -1749,39 +1749,6 @@ la instantánea utilizada por el sync interrumpido. Un sync posterior puede reemplazar esa instantánea, haciendo que las versiones intermedias de los archivos no estén disponibles para la recuperación. -9.1 Copias de seguridad externas con instantáneas del sistema de archivos (External backups with filesystem snapshots) ----------------------------------------------------------------------------------------------------------------------- - -Cuando las instantáneas del sistema de archivos están habilitadas, SnapRAID -accede o crea instantáneas en la raíz de cada subvolumen, volumen o conjunto -de datos (dataset). - -Una forma sencilla de evitar interferencias con herramientas externas es -apuntar la configuración `data` a un subdirectorio normal dentro del -subvolumen o volumen (por ejemplo, `/mnt/subvol/data` en lugar de -`/mnt/subvol`). SnapRAID siempre crea el directorio de instantáneas en la -raíz del subvolumen (por ejemplo, `/mnt/subvol/.snapraid/`), dejando limpio -el subdirectorio de datos para que las instantáneas permanezcan fuera e -invisibles para las herramientas de copia de seguridad que operan dentro -de él. - -De lo contrario, si el directorio `data` configurado es en sí mismo la raíz -del subvolumen o volumen, SnapRAID crea un directorio `.snapraid/` -directamente dentro de él (en Btrfs, Bcachefs y NTFS). Aunque SnapRAID -excluye automáticamente este directorio de su propio escaneo, las -herramientas externas de copia de seguridad o sincronización de archivos lo -encontrarán. Si no se excluye, pueden acceder a las instantáneas o enlaces -de directorio y copiar todo el árbol de datos congelado, duplicando datos y -agotando el almacenamiento de respaldo. Configure siempre las herramientas -de copia de seguridad externas para excluir el directorio `.snapraid/`. - -En ZFS, las instantáneas se exponen a través del directorio virtual estándar -`.zfs/snapshot/` en lugar de `.snapraid/`. De forma predeterminada -(`snapdir=hidden`), ZFS oculta este directorio de los listados, por lo que -las herramientas externas de copia de seguridad no lo recorrerán. Si ha -habilitado `snapdir=visible`, asegúrese de que sus herramientas de copia -de seguridad excluyan el directorio `.zfs/`. - 10 PATRONES (PATTERN) ===================== diff --git a/doc/snapraid-fr.1 b/doc/snapraid-fr.1 index 40471034..03482ca6 100644 --- a/doc/snapraid-fr.1 +++ b/doc/snapraid-fr.1 @@ -2071,36 +2071,6 @@ avec succès. La parité peut encore contenir des blocs calculés à partir de l\'instantané utilisé par le sync interrompu. Un sync ultérieur peut remplacer cet instantané, rendant les versions intermédiaires des fichiers indisponibles pour la récupération. -.SS Sauvegardes externes avec les instantanés de système de fichiers (External backups with filesystem snapshots) -Lorsque les instantanés de système de fichiers sont activés, SnapRAID accède -aux instantanés ou les crée à la racine de chaque sous\-volume, volume ou -jeu de données (dataset). -.PP -Un moyen simple d\'éviter toute interférence avec les outils externes -consiste à faire pointer la configuration \`data\` vers un sous\-répertoire -ordinaire au sein du sous\-volume ou du volume (par exemple -\`/mnt/subvol/data\` au lieu de \`/mnt/subvol\`). SnapRAID crée toujours le -répertoire d\'instantanés à la racine du sous\-volume (par exemple -\`/mnt/subvol/.snapraid/\`), laissant le sous\-répertoire de données propre de -sorte que les instantanés restent à l\'extérieur et invisibles pour les -outils de sauvegarde opérant à l\'intérieur. -.PP -Sinon, si votre répertoire \`data\` configuré est lui\-même la racine du -sous\-volume ou du volume, SnapRAID crée un répertoire \`.snapraid/\` -directement à l\'intérieur (sur Btrfs, Bcachefs et NTFS). Bien que SnapRAID -exclue automatiquement ce répertoire de ses propres analyses, les outils -de sauvegarde externes ou les synchroniseurs de fichiers le rencontreront. -S\'il n\'est pas exclu, ils peuvent traverser les instantanés ou les liens de -répertoires et copier l\'ensemble de l\'arborescence figée, dupliquant les -données et risquant d\'épuiser l\'espace de sauvegarde. Configurez toujours -les outils de sauvegarde externes pour exclure le répertoire \`.snapraid/\`. -.PP -Sur ZFS, les instantanés sont exposés via le répertoire virtuel standard -\`.zfs/snapshot/\` plutôt que \`.snapraid/\`. Par défaut (\`snapdir=hidden\`), ZFS -masque ce répertoire des listes de fichiers, de sorte que les outils de -sauvegarde externes ne le parcourront pas. Si vous avez activé -\`snapdir=visible\`, veillez à ce que vos outils de sauvegarde excluent le -répertoire \`.zfs/\`. .SH MOTIF (PATTERN) Les motifs offrent un moyen flexible de filtrer les fichiers à inclure ou à exclure. En utilisant des caractères de \[dq]globbing\[dq], vous pouvez définir des règles qui diff --git a/doc/snapraid-fr.d b/doc/snapraid-fr.d index 513d0664..39b5c8d1 100644 --- a/doc/snapraid-fr.d +++ b/doc/snapraid-fr.d @@ -1676,37 +1676,6 @@ Cycle de vie des instantanés (Snapshots Lifecycle) cet instantané, rendant les versions intermédiaires des fichiers indisponibles pour la récupération. - Sauvegardes externes avec les instantanés de système de fichiers (External backups with filesystem snapshots) - Lorsque les instantanés de système de fichiers sont activés, SnapRAID accède - aux instantanés ou les crée à la racine de chaque sous-volume, volume ou - jeu de données (dataset). - - Un moyen simple d'éviter toute interférence avec les outils externes - consiste à faire pointer la configuration `data` vers un sous-répertoire - ordinaire au sein du sous-volume ou du volume (par exemple - `/mnt/subvol/data` au lieu de `/mnt/subvol`). SnapRAID crée toujours le - répertoire d'instantanés à la racine du sous-volume (par exemple - `/mnt/subvol/.snapraid/`), laissant le sous-répertoire de données propre de - sorte que les instantanés restent à l'extérieur et invisibles pour les - outils de sauvegarde opérant à l'intérieur. - - Sinon, si votre répertoire `data` configuré est lui-même la racine du - sous-volume ou du volume, SnapRAID crée un répertoire `.snapraid/` - directement à l'intérieur (sur Btrfs, Bcachefs et NTFS). Bien que SnapRAID - exclue automatiquement ce répertoire de ses propres analyses, les outils - de sauvegarde externes ou les synchroniseurs de fichiers le rencontreront. - S'il n'est pas exclu, ils peuvent traverser les instantanés ou les liens de - répertoires et copier l'ensemble de l'arborescence figée, dupliquant les - données et risquant d'épuiser l'espace de sauvegarde. Configurez toujours - les outils de sauvegarde externes pour exclure le répertoire `.snapraid/`. - - Sur ZFS, les instantanés sont exposés via le répertoire virtuel standard - `.zfs/snapshot/` plutôt que `.snapraid/`. Par défaut (`snapdir=hidden`), ZFS - masque ce répertoire des listes de fichiers, de sorte que les outils de - sauvegarde externes ne le parcourront pas. Si vous avez activé - `snapdir=visible`, veillez à ce que vos outils de sauvegarde excluent le - répertoire `.zfs/`. - Motif (Pattern) Les motifs offrent un moyen flexible de filtrer les fichiers à inclure ou à exclure. En utilisant des caractères de "globbing", vous pouvez définir des règles qui diff --git a/doc/snapraid-fr.txt b/doc/snapraid-fr.txt index c69cc2ed..9169b1fc 100644 --- a/doc/snapraid-fr.txt +++ b/doc/snapraid-fr.txt @@ -1786,39 +1786,6 @@ l'instantané utilisé par le sync interrompu. Un sync ultérieur peut remplacer cet instantané, rendant les versions intermédiaires des fichiers indisponibles pour la récupération. -9.1 Sauvegardes externes avec les instantanés de système de fichiers (External backups with filesystem snapshots) ------------------------------------------------------------------------------------------------------------------ - -Lorsque les instantanés de système de fichiers sont activés, SnapRAID accède -aux instantanés ou les crée à la racine de chaque sous-volume, volume ou -jeu de données (dataset). - -Un moyen simple d'éviter toute interférence avec les outils externes -consiste à faire pointer la configuration `data` vers un sous-répertoire -ordinaire au sein du sous-volume ou du volume (par exemple -`/mnt/subvol/data` au lieu de `/mnt/subvol`). SnapRAID crée toujours le -répertoire d'instantanés à la racine du sous-volume (par exemple -`/mnt/subvol/.snapraid/`), laissant le sous-répertoire de données propre de -sorte que les instantanés restent à l'extérieur et invisibles pour les -outils de sauvegarde opérant à l'intérieur. - -Sinon, si votre répertoire `data` configuré est lui-même la racine du -sous-volume ou du volume, SnapRAID crée un répertoire `.snapraid/` -directement à l'intérieur (sur Btrfs, Bcachefs et NTFS). Bien que SnapRAID -exclue automatiquement ce répertoire de ses propres analyses, les outils -de sauvegarde externes ou les synchroniseurs de fichiers le rencontreront. -S'il n'est pas exclu, ils peuvent traverser les instantanés ou les liens de -répertoires et copier l'ensemble de l'arborescence figée, dupliquant les -données et risquant d'épuiser l'espace de sauvegarde. Configurez toujours -les outils de sauvegarde externes pour exclure le répertoire `.snapraid/`. - -Sur ZFS, les instantanés sont exposés via le répertoire virtuel standard -`.zfs/snapshot/` plutôt que `.snapraid/`. Par défaut (`snapdir=hidden`), ZFS -masque ce répertoire des listes de fichiers, de sorte que les outils de -sauvegarde externes ne le parcourront pas. Si vous avez activé -`snapdir=visible`, veillez à ce que vos outils de sauvegarde excluent le -répertoire `.zfs/`. - 10 MOTIF (PATTERN) ================== diff --git a/doc/snapraid-it.1 b/doc/snapraid-it.1 index 5b5326df..ab7aff8d 100644 --- a/doc/snapraid-it.1 +++ b/doc/snapraid-it.1 @@ -2055,34 +2055,6 @@ con successo. La parità potrebbe ancora contenere blocchi calcolati dallo snapshot utilizzato dal sync interrotto. Un sync successivo potrebbe sostituire tale snapshot, rendendo le versioni intermedie dei file non disponibili per il recupero. -.SS Backup esterni con snapshot del filesystem (External backups with filesystem snapshots) -Quando gli snapshot del filesystem sono abilitati, SnapRAID accede o crea -gli snapshot alla radice di ciascun sottovolume, volume o dataset di dati. -.PP -Un modo semplice per evitare interferenze con gli strumenti esterni consiste -nel far puntare la configurazione \`data\` a una sottodirectory ordinaria -all\'interno del subvolume o del volume (ad esempio \`/mnt/subvol/data\` invece -di \`/mnt/subvol\`). SnapRAID crea sempre la directory degli snapshot alla -radice del subvolume (ad esempio \`/mnt/subvol/.snapraid/\`), lasciando pulita -la sottodirectory dei dati in modo che gli snapshot rimangano all\'esterno -e invisibili agli strumenti di backup che operano al suo interno. -.PP -Altrimenti, se la directory \`data\` configurata è essa stessa la radice del -subvolume o del volume, SnapRAID crea una directory \`.snapraid/\` direttamente -al suo interno (su Btrfs, Bcachefs e NTFS). Sebbene SnapRAID escluda -automaticamente questa directory dalla propria scansione, gli strumenti di -backup esterni o i programmi di sincronizzazione dei file la incontreranno. -Se non viene esclusa, potrebbero entrare negli snapshot o nei collegamenti -di directory e copiare l\'intero albero dei dati congelato, duplicando i dati -ed esaurendo lo spazio di archiviazione dei backup. Configurare sempre gli -strumenti di backup esterni in modo da escludere la directory \`.snapraid/\`. -.PP -Su ZFS, gli snapshot sono esposti tramite la directory virtuale standard -\`.zfs/snapshot/\` anziché \`.snapraid/\`. Di default (\`snapdir=hidden\`), ZFS -nasconde questa directory dall\'elenco delle directory, quindi gli strumenti -di backup esterni non la attraverseranno. Se è stato abilitato -\`snapdir=visible\`, assicurarsi che gli strumenti di backup escludano la -directory \`.zfs/\`. .SH PATTERN I pattern offrono un modo flessibile per filtrare i file da includere o escludere. Usando i caratteri globbing, è possibile definire regole che diff --git a/doc/snapraid-it.d b/doc/snapraid-it.d index e69417a5..a300db88 100644 --- a/doc/snapraid-it.d +++ b/doc/snapraid-it.d @@ -1649,35 +1649,6 @@ Ciclo di vita degli snapshot (Snapshots Lifecycle) tale snapshot, rendendo le versioni intermedie dei file non disponibili per il recupero. - Backup esterni con snapshot del filesystem (External backups with filesystem snapshots) - Quando gli snapshot del filesystem sono abilitati, SnapRAID accede o crea - gli snapshot alla radice di ciascun sottovolume, volume o dataset di dati. - - Un modo semplice per evitare interferenze con gli strumenti esterni consiste - nel far puntare la configurazione `data` a una sottodirectory ordinaria - all'interno del subvolume o del volume (ad esempio `/mnt/subvol/data` invece - di `/mnt/subvol`). SnapRAID crea sempre la directory degli snapshot alla - radice del subvolume (ad esempio `/mnt/subvol/.snapraid/`), lasciando pulita - la sottodirectory dei dati in modo che gli snapshot rimangano all'esterno - e invisibili agli strumenti di backup che operano al suo interno. - - Altrimenti, se la directory `data` configurata è essa stessa la radice del - subvolume o del volume, SnapRAID crea una directory `.snapraid/` direttamente - al suo interno (su Btrfs, Bcachefs e NTFS). Sebbene SnapRAID escluda - automaticamente questa directory dalla propria scansione, gli strumenti di - backup esterni o i programmi di sincronizzazione dei file la incontreranno. - Se non viene esclusa, potrebbero entrare negli snapshot o nei collegamenti - di directory e copiare l'intero albero dei dati congelato, duplicando i dati - ed esaurendo lo spazio di archiviazione dei backup. Configurare sempre gli - strumenti di backup esterni in modo da escludere la directory `.snapraid/`. - - Su ZFS, gli snapshot sono esposti tramite la directory virtuale standard - `.zfs/snapshot/` anziché `.snapraid/`. Di default (`snapdir=hidden`), ZFS - nasconde questa directory dall'elenco delle directory, quindi gli strumenti - di backup esterni non la attraverseranno. Se è stato abilitato - `snapdir=visible`, assicurarsi che gli strumenti di backup escludano la - directory `.zfs/`. - Pattern I pattern offrono un modo flessibile per filtrare i file da includere o escludere. Usando i caratteri globbing, è possibile definire regole che diff --git a/doc/snapraid-it.txt b/doc/snapraid-it.txt index 711a98cf..bf0b3c36 100644 --- a/doc/snapraid-it.txt +++ b/doc/snapraid-it.txt @@ -1759,37 +1759,6 @@ snapshot utilizzato dal sync interrotto. Un sync successivo potrebbe sostituire tale snapshot, rendendo le versioni intermedie dei file non disponibili per il recupero. -9.1 Backup esterni con snapshot del filesystem (External backups with filesystem snapshots) -------------------------------------------------------------------------------------------- - -Quando gli snapshot del filesystem sono abilitati, SnapRAID accede o crea -gli snapshot alla radice di ciascun sottovolume, volume o dataset di dati. - -Un modo semplice per evitare interferenze con gli strumenti esterni consiste -nel far puntare la configurazione `data` a una sottodirectory ordinaria -all'interno del subvolume o del volume (ad esempio `/mnt/subvol/data` invece -di `/mnt/subvol`). SnapRAID crea sempre la directory degli snapshot alla -radice del subvolume (ad esempio `/mnt/subvol/.snapraid/`), lasciando pulita -la sottodirectory dei dati in modo che gli snapshot rimangano all'esterno -e invisibili agli strumenti di backup che operano al suo interno. - -Altrimenti, se la directory `data` configurata è essa stessa la radice del -subvolume o del volume, SnapRAID crea una directory `.snapraid/` direttamente -al suo interno (su Btrfs, Bcachefs e NTFS). Sebbene SnapRAID escluda -automaticamente questa directory dalla propria scansione, gli strumenti di -backup esterni o i programmi di sincronizzazione dei file la incontreranno. -Se non viene esclusa, potrebbero entrare negli snapshot o nei collegamenti -di directory e copiare l'intero albero dei dati congelato, duplicando i dati -ed esaurendo lo spazio di archiviazione dei backup. Configurare sempre gli -strumenti di backup esterni in modo da escludere la directory `.snapraid/`. - -Su ZFS, gli snapshot sono esposti tramite la directory virtuale standard -`.zfs/snapshot/` anziché `.snapraid/`. Di default (`snapdir=hidden`), ZFS -nasconde questa directory dall'elenco delle directory, quindi gli strumenti -di backup esterni non la attraverseranno. Se è stato abilitato -`snapdir=visible`, assicurarsi che gli strumenti di backup escludano la -directory `.zfs/`. - 10 PATTERN ========== diff --git a/doc/snapraid-ja.1 b/doc/snapraid-ja.1 index a11a9d20..efdca0e8 100644 --- a/doc/snapraid-ja.1 +++ b/doc/snapraid-ja.1 @@ -1833,31 +1833,6 @@ ZFSでは標準の \`.zfs/snapshot/\` が使用されます。 パリティには、中断された sync で使用されたスナップショットから計算されたブロックが まだ含まれている可能性があります。その後の sync によりそのスナップショットが 置き換えられ、中間バージョンのファイルが復元できなくなる可能性があります。 -.SS ファイルシステムスナップショットと外部バックアップ (External backups with filesystem snapshots) -ファイルシステムスナップショットが有効な場合、SnapRAIDは各データサブボリューム、 -ボリューム、またはデータセットのルートにあるスナップショットにアクセスまたは作成します。 -.PP -外部ツールとの干渉を避ける簡単な方法は、\`data\` 設定をサブボリュームまたは -ボリューム内の通常のサブディレクトリ(例えば \`/mnt/subvol\` の代わりに -\`/mnt/subvol/data\`)に指定することです。SnapRAIDは常にサブボリュームのルートに -スナップショットディレクトリを作成するため(例えば \`/mnt/subvol/.snapraid/\`)、 -データサブディレクトリがクリーンに保たれ、スナップショットはそのディレクトリ内で -動作するバックアップツールから見えない状態のままになります。 -.PP -一方、設定された \`data\` ディレクトリ自体がサブボリュームまたはボリュームの -ルートである場合、SnapRAIDはその直下に \`.snapraid/\` ディレクトリを作成します(Btrfs、 -Bcachefs、およびNTFS)。SnapRAID自身のスキャンからは自動的に除外されますが、 -外部のバックアップツールやファイル同期ツールはそのディレクトリを検出します。 -除外設定を行わない場合、スナップショットやディレクトリリンク内を走査して凍結された -データツリー全体をコピーしてしまい、データが重複してバックアップストレージを使い果たす -可能性があります。外部バックアップツールでは、必ず \`.snapraid/\` ディレクトリを -除外するように設定してください。 -.PP -ZFSでは、スナップショットは \`.snapraid/\` ではなく標準の仮想ディレクトリ -\`.zfs/snapshot/\` 経由で公開されます。デフォルト(\`snapdir=hidden\`)では、ZFSはこの -ディレクトリを一覧表示から隠すため、外部バックアップツールが走査することはありません。 -\`snapdir=visible\` を有効にしている場合は、バックアップツールが \`.zfs/\` ディレクトリを -除外するように設定してください。 .SH パターン (PATTERN) パターンは、含めるファイルまたは除外するファイルをフィルタリングするための柔軟な方法を 提供します。グロブ文字を使用することで、すべてのパスを手動でリストすることなく、 diff --git a/doc/snapraid-ja.d b/doc/snapraid-ja.d index 58a996b7..3931aefe 100644 --- a/doc/snapraid-ja.d +++ b/doc/snapraid-ja.d @@ -1438,32 +1438,6 @@ Name{number} まだ含まれている可能性があります。その後の sync によりそのスナップショットが 置き換えられ、中間バージョンのファイルが復元できなくなる可能性があります。 - ファイルシステムスナップショットと外部バックアップ (External backups with filesystem snapshots) - ファイルシステムスナップショットが有効な場合、SnapRAIDは各データサブボリューム、 - ボリューム、またはデータセットのルートにあるスナップショットにアクセスまたは作成します。 - - 外部ツールとの干渉を避ける簡単な方法は、`data` 設定をサブボリュームまたは - ボリューム内の通常のサブディレクトリ(例えば `/mnt/subvol` の代わりに - `/mnt/subvol/data`)に指定することです。SnapRAIDは常にサブボリュームのルートに - スナップショットディレクトリを作成するため(例えば `/mnt/subvol/.snapraid/`)、 - データサブディレクトリがクリーンに保たれ、スナップショットはそのディレクトリ内で - 動作するバックアップツールから見えない状態のままになります。 - - 一方、設定された `data` ディレクトリ自体がサブボリュームまたはボリュームの - ルートである場合、SnapRAIDはその直下に `.snapraid/` ディレクトリを作成します(Btrfs、 - Bcachefs、およびNTFS)。SnapRAID自身のスキャンからは自動的に除外されますが、 - 外部のバックアップツールやファイル同期ツールはそのディレクトリを検出します。 - 除外設定を行わない場合、スナップショットやディレクトリリンク内を走査して凍結された - データツリー全体をコピーしてしまい、データが重複してバックアップストレージを使い果たす - 可能性があります。外部バックアップツールでは、必ず `.snapraid/` ディレクトリを - 除外するように設定してください。 - - ZFSでは、スナップショットは `.snapraid/` ではなく標準の仮想ディレクトリ - `.zfs/snapshot/` 経由で公開されます。デフォルト(`snapdir=hidden`)では、ZFSはこの - ディレクトリを一覧表示から隠すため、外部バックアップツールが走査することはありません。 - `snapdir=visible` を有効にしている場合は、バックアップツールが `.zfs/` ディレクトリを - 除外するように設定してください。 - パターン (Pattern) パターンは、含めるファイルまたは除外するファイルをフィルタリングするための柔軟な方法を 提供します。グロブ文字を使用することで、すべてのパスを手動でリストすることなく、 diff --git a/doc/snapraid-ja.txt b/doc/snapraid-ja.txt index 580201d5..f8b28a74 100644 --- a/doc/snapraid-ja.txt +++ b/doc/snapraid-ja.txt @@ -1548,34 +1548,6 @@ ZFSでは標準の `.zfs/snapshot/` が使用されます。 まだ含まれている可能性があります。その後の sync によりそのスナップショットが 置き換えられ、中間バージョンのファイルが復元できなくなる可能性があります。 -9.1 ファイルシステムスナップショットと外部バックアップ (External backups with filesystem snapshots) --------------------------------------------------------------------------- - -ファイルシステムスナップショットが有効な場合、SnapRAIDは各データサブボリューム、 -ボリューム、またはデータセットのルートにあるスナップショットにアクセスまたは作成します。 - -外部ツールとの干渉を避ける簡単な方法は、`data` 設定をサブボリュームまたは -ボリューム内の通常のサブディレクトリ(例えば `/mnt/subvol` の代わりに -`/mnt/subvol/data`)に指定することです。SnapRAIDは常にサブボリュームのルートに -スナップショットディレクトリを作成するため(例えば `/mnt/subvol/.snapraid/`)、 -データサブディレクトリがクリーンに保たれ、スナップショットはそのディレクトリ内で -動作するバックアップツールから見えない状態のままになります。 - -一方、設定された `data` ディレクトリ自体がサブボリュームまたはボリュームの -ルートである場合、SnapRAIDはその直下に `.snapraid/` ディレクトリを作成します(Btrfs、 -Bcachefs、およびNTFS)。SnapRAID自身のスキャンからは自動的に除外されますが、 -外部のバックアップツールやファイル同期ツールはそのディレクトリを検出します。 -除外設定を行わない場合、スナップショットやディレクトリリンク内を走査して凍結された -データツリー全体をコピーしてしまい、データが重複してバックアップストレージを使い果たす -可能性があります。外部バックアップツールでは、必ず `.snapraid/` ディレクトリを -除外するように設定してください。 - -ZFSでは、スナップショットは `.snapraid/` ではなく標準の仮想ディレクトリ -`.zfs/snapshot/` 経由で公開されます。デフォルト(`snapdir=hidden`)では、ZFSはこの -ディレクトリを一覧表示から隠すため、外部バックアップツールが走査することはありません。 -`snapdir=visible` を有効にしている場合は、バックアップツールが `.zfs/` ディレクトリを -除外するように設定してください。 - 10 パターン (PATTERN) ================= diff --git a/doc/snapraid-ko.1 b/doc/snapraid-ko.1 index 406369b0..c5af4752 100644 --- a/doc/snapraid-ko.1 +++ b/doc/snapraid-ko.1 @@ -1877,28 +1877,6 @@ ZFS에서는 표준 \`.zfs/snapshot/\` 이 사용됩니다. 패리티에는 중단된 sync에서 사용된 스냅샷에서 계산된 블록이 여전히 포함되어 있을 수 있습니다. 이후의 sync로 인해 해당 스냅샷이 교체되어 복구에 필요한 중간 버전의 파일을 사용할 수 없게 될 수 있습니다. -.SS 파일 시스템 스냅샷을 사용한 외부 백업 (External backups with filesystem snapshots) -파일 시스템 스냅샷이 활성화되면 SnapRAID는 각 데이터 서브볼륨, 볼륨 또는 -데이터셋의 루트에서 스냅샷에 접근하거나 생성합니다. -.PP -외부 도구와의 간섭을 피하는 간단한 방법은 서브볼륨이나 볼륨 내의 일반 -하위 디렉터리(예: \`/mnt/subvol\` 대신 \`/mnt/subvol/data\`)를 가리키도록 -\`data\` 를 설정하는 것입니다. SnapRAID는 항상 서브볼륨 루트에 스냅샷 디렉터리를 -생성하므로(예: \`/mnt/subvol/.snapraid/\`), 데이터 하위 디렉터리를 깨끗하게 -유지하여 해당 디렉터리 내에서 작동하는 백업 도구에 스냅샷이 노출되지 않도록 합니다. -.PP -그렇지 않고 설정된 \`data\` 디렉터리 자체가 서브볼륨이나 볼륨 루트인 경우, -SnapRAID는 해당 디렉터리 바로 안에 \`.snapraid/\` 디렉터리를 생성합니다(Btrfs, -Bcachefs 및 NTFS). SnapRAID 자체 스캔에서는 이 디렉터리가 자동으로 제외되지만, -외부 백업 도구나 파일 동기화 도구는 이를 발견하게 됩니다. 제외하지 않으면 -스냅샷이나 디렉터리 링크를 탐색하여 동결된 전체 데이터 트리를 복사함으로써 -데이터를 중복 생성하고 백업 저장 공간을 고갈시킬 수 있습니다. 항상 외부 백업 -도구에서 \`.snapraid/\` 디렉터리를 제외하도록 구성하십시오. -.PP -ZFS에서는 스냅샷이 \`.snapraid/\` 대신 표준 가상 디렉터리인 \`.zfs/snapshot/\` 을 -통해 노출됩니다. 기본값(\`snapdir=hidden\`)에서는 ZFS가 디렉터리 목록에서 이 -디렉터리를 숨기므로 외부 백업 도구가 이를 탐색하지 않습니다. \`snapdir=visible\` 로 -설정한 경우 백업 도구에서 \`.zfs/\` 디렉터리를 제외하도록 하십시오. .SH 패턴 (PATTERN) 패턴은 포함하거나 제외할 파일을 필터링하는 유연한 방법을 제공합니다. 글로빙(globbing) 문자를 사용하면 모든 경로를 일일이 나열하지 않고도 diff --git a/doc/snapraid-ko.d b/doc/snapraid-ko.d index 181827b1..896fc3f7 100644 --- a/doc/snapraid-ko.d +++ b/doc/snapraid-ko.d @@ -1482,29 +1482,6 @@ Name{number} 여전히 포함되어 있을 수 있습니다. 이후의 sync로 인해 해당 스냅샷이 교체되어 복구에 필요한 중간 버전의 파일을 사용할 수 없게 될 수 있습니다. - 파일 시스템 스냅샷을 사용한 외부 백업 (External backups with filesystem snapshots) - 파일 시스템 스냅샷이 활성화되면 SnapRAID는 각 데이터 서브볼륨, 볼륨 또는 - 데이터셋의 루트에서 스냅샷에 접근하거나 생성합니다. - - 외부 도구와의 간섭을 피하는 간단한 방법은 서브볼륨이나 볼륨 내의 일반 - 하위 디렉터리(예: `/mnt/subvol` 대신 `/mnt/subvol/data`)를 가리키도록 - `data` 를 설정하는 것입니다. SnapRAID는 항상 서브볼륨 루트에 스냅샷 디렉터리를 - 생성하므로(예: `/mnt/subvol/.snapraid/`), 데이터 하위 디렉터리를 깨끗하게 - 유지하여 해당 디렉터리 내에서 작동하는 백업 도구에 스냅샷이 노출되지 않도록 합니다. - - 그렇지 않고 설정된 `data` 디렉터리 자체가 서브볼륨이나 볼륨 루트인 경우, - SnapRAID는 해당 디렉터리 바로 안에 `.snapraid/` 디렉터리를 생성합니다(Btrfs, - Bcachefs 및 NTFS). SnapRAID 자체 스캔에서는 이 디렉터리가 자동으로 제외되지만, - 외부 백업 도구나 파일 동기화 도구는 이를 발견하게 됩니다. 제외하지 않으면 - 스냅샷이나 디렉터리 링크를 탐색하여 동결된 전체 데이터 트리를 복사함으로써 - 데이터를 중복 생성하고 백업 저장 공간을 고갈시킬 수 있습니다. 항상 외부 백업 - 도구에서 `.snapraid/` 디렉터리를 제외하도록 구성하십시오. - - ZFS에서는 스냅샷이 `.snapraid/` 대신 표준 가상 디렉터리인 `.zfs/snapshot/` 을 - 통해 노출됩니다. 기본값(`snapdir=hidden`)에서는 ZFS가 디렉터리 목록에서 이 - 디렉터리를 숨기므로 외부 백업 도구가 이를 탐색하지 않습니다. `snapdir=visible` 로 - 설정한 경우 백업 도구에서 `.zfs/` 디렉터리를 제외하도록 하십시오. - 패턴 (Pattern) 패턴은 포함하거나 제외할 파일을 필터링하는 유연한 방법을 제공합니다. 글로빙(globbing) 문자를 사용하면 모든 경로를 일일이 나열하지 않고도 diff --git a/doc/snapraid-ko.txt b/doc/snapraid-ko.txt index 0bc5dd8b..db1e385d 100644 --- a/doc/snapraid-ko.txt +++ b/doc/snapraid-ko.txt @@ -1592,31 +1592,6 @@ ZFS에서는 표준 `.zfs/snapshot/` 이 사용됩니다. 여전히 포함되어 있을 수 있습니다. 이후의 sync로 인해 해당 스냅샷이 교체되어 복구에 필요한 중간 버전의 파일을 사용할 수 없게 될 수 있습니다. -9.1 파일 시스템 스냅샷을 사용한 외부 백업 (External backups with filesystem snapshots) ----------------------------------------------------------------------- - -파일 시스템 스냅샷이 활성화되면 SnapRAID는 각 데이터 서브볼륨, 볼륨 또는 -데이터셋의 루트에서 스냅샷에 접근하거나 생성합니다. - -외부 도구와의 간섭을 피하는 간단한 방법은 서브볼륨이나 볼륨 내의 일반 -하위 디렉터리(예: `/mnt/subvol` 대신 `/mnt/subvol/data`)를 가리키도록 -`data` 를 설정하는 것입니다. SnapRAID는 항상 서브볼륨 루트에 스냅샷 디렉터리를 -생성하므로(예: `/mnt/subvol/.snapraid/`), 데이터 하위 디렉터리를 깨끗하게 -유지하여 해당 디렉터리 내에서 작동하는 백업 도구에 스냅샷이 노출되지 않도록 합니다. - -그렇지 않고 설정된 `data` 디렉터리 자체가 서브볼륨이나 볼륨 루트인 경우, -SnapRAID는 해당 디렉터리 바로 안에 `.snapraid/` 디렉터리를 생성합니다(Btrfs, -Bcachefs 및 NTFS). SnapRAID 자체 스캔에서는 이 디렉터리가 자동으로 제외되지만, -외부 백업 도구나 파일 동기화 도구는 이를 발견하게 됩니다. 제외하지 않으면 -스냅샷이나 디렉터리 링크를 탐색하여 동결된 전체 데이터 트리를 복사함으로써 -데이터를 중복 생성하고 백업 저장 공간을 고갈시킬 수 있습니다. 항상 외부 백업 -도구에서 `.snapraid/` 디렉터리를 제외하도록 구성하십시오. - -ZFS에서는 스냅샷이 `.snapraid/` 대신 표준 가상 디렉터리인 `.zfs/snapshot/` 을 -통해 노출됩니다. 기본값(`snapdir=hidden`)에서는 ZFS가 디렉터리 목록에서 이 -디렉터리를 숨기므로 외부 백업 도구가 이를 탐색하지 않습니다. `snapdir=visible` 로 -설정한 경우 백업 도구에서 `.zfs/` 디렉터리를 제외하도록 하십시오. - 10 패턴 (PATTERN) =============== diff --git a/doc/snapraid-pl.1 b/doc/snapraid-pl.1 index f58b449d..be4409c7 100644 --- a/doc/snapraid-pl.1 +++ b/doc/snapraid-pl.1 @@ -1954,36 +1954,6 @@ usuwania, zastępowania lub przenoszenia plików do momentu pomyślnego ukończe kolejnej operacji \`sync\`. Parzystość może nadal zawierać bloki obliczone na podstawie migawki użytej przez przerwaną synchronizację. Późniejsza synchronizacja może zastąpić tę migawkę, uniemożliwiając odzyskanie pośrednich wersji plików. -.SS Kopie zapasowe z migawkami systemu plików (External backups with filesystem snapshots) -Gdy migawki systemu plików są włączone, SnapRAID uzyskuje dostęp do migawek -lub tworzy je w katalogu głównym każdego podwoluminu danych, woluminu lub -zestawu danych (dataset). -.PP -Prostym sposobem na uniknięcie jakichkolwiek konfliktów z zewnętrznymi -narzędziami jest wskazanie w konfiguracji \`data\` zwykłego podkatalogu -wewnątrz podwoluminu lub woluminu (na przykład \`/mnt/subvol/data\` zamiast -\`/mnt/subvol\`). SnapRAID zawsze tworzy katalog migawek w katalogu głównym -podwoluminu (na przykład \`/mnt/subvol/.snapraid/\`), pozostawiając podkatalog -danych czysty, dzięki czemu migawki pozostają na zewnątrz i są niewidoczne -dla narzędzi do tworzenia kopii zapasowych działających w tym katalogu. -.PP -W przeciwnym razie, jeśli skonfigurowany katalog \`data\` jest sam w sobie -katalogiem głównym podwoluminu lub woluminu, SnapRAID utworzy katalog -\`.snapraid/\` bezpośrednio w nim (w systemach Btrfs, Bcachefs i NTFS). -Chociaż SnapRAID automatycznie wyklucza ten katalog z własnego skanowania, -zewnętrzne narzędzia do tworzenia kopii zapasowych lub synchronizacji -plików napotkają go. Jeśli nie zostanie wykluczony, mogą wejść do migawek -lub dowiązań katalogów i skopiować całe zamrożone drzewo danych, duplikując -dane i wyczerpując przestrzeń dyskową kopii zapasowej. Zawsze konfiguruj -zewnętrzne narzędzia do tworzenia kopii zapasowych tak, aby wykluczały -katalog \`.snapraid/\`. -.PP -W systemie ZFS migawki są udostępniane za pośrednictwem standardowego -katalogu wirtualnego \`.zfs/snapshot/\` zamiast \`.snapraid/\`. Domyślnie -(\`snapdir=hidden\`) system ZFS ukrywa ten katalog na listach katalogów, więc -zewnętrzne narzędzia do tworzenia kopii zapasowych go nie przeskanują. Jeśli -włączono opcję \`snapdir=visible\`, upewnij się, że narzędzia do tworzenia kopii -zapasowych wykluczają katalog \`.zfs/\`. .SH WZORCE (PATTERN) Wzorce zapewniają elastyczny sposób filtrowania plików do uwzględnienia lub wykluczenia. Używając znaków globbingowych, można zdefiniować reguły, które diff --git a/doc/snapraid-pl.d b/doc/snapraid-pl.d index 5994af30..164a97ab 100644 --- a/doc/snapraid-pl.d +++ b/doc/snapraid-pl.d @@ -1559,37 +1559,6 @@ Cykl życia migawek (Snapshots Lifecycle) migawki użytej przez przerwaną synchronizację. Późniejsza synchronizacja może zastąpić tę migawkę, uniemożliwiając odzyskanie pośrednich wersji plików. - Kopie zapasowe z migawkami systemu plików (External backups with filesystem snapshots) - Gdy migawki systemu plików są włączone, SnapRAID uzyskuje dostęp do migawek - lub tworzy je w katalogu głównym każdego podwoluminu danych, woluminu lub - zestawu danych (dataset). - - Prostym sposobem na uniknięcie jakichkolwiek konfliktów z zewnętrznymi - narzędziami jest wskazanie w konfiguracji `data` zwykłego podkatalogu - wewnątrz podwoluminu lub woluminu (na przykład `/mnt/subvol/data` zamiast - `/mnt/subvol`). SnapRAID zawsze tworzy katalog migawek w katalogu głównym - podwoluminu (na przykład `/mnt/subvol/.snapraid/`), pozostawiając podkatalog - danych czysty, dzięki czemu migawki pozostają na zewnątrz i są niewidoczne - dla narzędzi do tworzenia kopii zapasowych działających w tym katalogu. - - W przeciwnym razie, jeśli skonfigurowany katalog `data` jest sam w sobie - katalogiem głównym podwoluminu lub woluminu, SnapRAID utworzy katalog - `.snapraid/` bezpośrednio w nim (w systemach Btrfs, Bcachefs i NTFS). - Chociaż SnapRAID automatycznie wyklucza ten katalog z własnego skanowania, - zewnętrzne narzędzia do tworzenia kopii zapasowych lub synchronizacji - plików napotkają go. Jeśli nie zostanie wykluczony, mogą wejść do migawek - lub dowiązań katalogów i skopiować całe zamrożone drzewo danych, duplikując - dane i wyczerpując przestrzeń dyskową kopii zapasowej. Zawsze konfiguruj - zewnętrzne narzędzia do tworzenia kopii zapasowych tak, aby wykluczały - katalog `.snapraid/`. - - W systemie ZFS migawki są udostępniane za pośrednictwem standardowego - katalogu wirtualnego `.zfs/snapshot/` zamiast `.snapraid/`. Domyślnie - (`snapdir=hidden`) system ZFS ukrywa ten katalog na listach katalogów, więc - zewnętrzne narzędzia do tworzenia kopii zapasowych go nie przeskanują. Jeśli - włączono opcję `snapdir=visible`, upewnij się, że narzędzia do tworzenia kopii - zapasowych wykluczają katalog `.zfs/`. - Wzorce (Pattern) Wzorce zapewniają elastyczny sposób filtrowania plików do uwzględnienia lub wykluczenia. Używając znaków globbingowych, można zdefiniować reguły, które diff --git a/doc/snapraid-pl.txt b/doc/snapraid-pl.txt index 8ad42b3c..8b7af044 100644 --- a/doc/snapraid-pl.txt +++ b/doc/snapraid-pl.txt @@ -1669,39 +1669,6 @@ kolejnej operacji `sync`. Parzystość może nadal zawierać bloki obliczone na migawki użytej przez przerwaną synchronizację. Późniejsza synchronizacja może zastąpić tę migawkę, uniemożliwiając odzyskanie pośrednich wersji plików. -9.1 Kopie zapasowe z migawkami systemu plików (External backups with filesystem snapshots) ------------------------------------------------------------------------------------------- - -Gdy migawki systemu plików są włączone, SnapRAID uzyskuje dostęp do migawek -lub tworzy je w katalogu głównym każdego podwoluminu danych, woluminu lub -zestawu danych (dataset). - -Prostym sposobem na uniknięcie jakichkolwiek konfliktów z zewnętrznymi -narzędziami jest wskazanie w konfiguracji `data` zwykłego podkatalogu -wewnątrz podwoluminu lub woluminu (na przykład `/mnt/subvol/data` zamiast -`/mnt/subvol`). SnapRAID zawsze tworzy katalog migawek w katalogu głównym -podwoluminu (na przykład `/mnt/subvol/.snapraid/`), pozostawiając podkatalog -danych czysty, dzięki czemu migawki pozostają na zewnątrz i są niewidoczne -dla narzędzi do tworzenia kopii zapasowych działających w tym katalogu. - -W przeciwnym razie, jeśli skonfigurowany katalog `data` jest sam w sobie -katalogiem głównym podwoluminu lub woluminu, SnapRAID utworzy katalog -`.snapraid/` bezpośrednio w nim (w systemach Btrfs, Bcachefs i NTFS). -Chociaż SnapRAID automatycznie wyklucza ten katalog z własnego skanowania, -zewnętrzne narzędzia do tworzenia kopii zapasowych lub synchronizacji -plików napotkają go. Jeśli nie zostanie wykluczony, mogą wejść do migawek -lub dowiązań katalogów i skopiować całe zamrożone drzewo danych, duplikując -dane i wyczerpując przestrzeń dyskową kopii zapasowej. Zawsze konfiguruj -zewnętrzne narzędzia do tworzenia kopii zapasowych tak, aby wykluczały -katalog `.snapraid/`. - -W systemie ZFS migawki są udostępniane za pośrednictwem standardowego -katalogu wirtualnego `.zfs/snapshot/` zamiast `.snapraid/`. Domyślnie -(`snapdir=hidden`) system ZFS ukrywa ten katalog na listach katalogów, więc -zewnętrzne narzędzia do tworzenia kopii zapasowych go nie przeskanują. Jeśli -włączono opcję `snapdir=visible`, upewnij się, że narzędzia do tworzenia kopii -zapasowych wykluczają katalog `.zfs/`. - 10 WZORCE (PATTERN) =================== diff --git a/doc/snapraid-pt.1 b/doc/snapraid-pt.1 index 42c6fa53..08ad589a 100644 --- a/doc/snapraid-pt.1 +++ b/doc/snapraid-pt.1 @@ -1992,36 +1992,6 @@ com sucesso. A paridade ainda pode conter blocos calculados a partir do instantâneo usado pelo sync interrompido. Um sync posterior pode substituir esse instantâneo, tornando as versões intermediárias dos arquivos indisponíveis para recuperação. -.SS Cópias de segurança externas com instantâneos do sistema de ficheiros (External backups with filesystem snapshots) -Quando os instantâneos do sistema de ficheiros estão ativados, o SnapRAID -acede ou cria instantâneos na raiz de cada subvolume, volume ou conjunto -de dados (dataset). -.PP -Uma forma simples de evitar qualquer interferência com ferramentas externas -é apontar a configuração \`data\` para um subdiretório normal dentro do -subvolume ou volume (por exemplo, \`/mnt/subvol/data\` em vez de \`/mnt/subvol\`). -O SnapRAID cria sempre o diretório de instantâneos na raiz do subvolume -(por exemplo, \`/mnt/subvol/.snapraid/\`), deixando o subdiretório de dados -limpo, de modo que os instantâneos permaneçam fora e invisíveis para as -ferramentas de cópia de segurança que operam dentro dele. -.PP -Caso contrário, se o diretório \`data\` configurado for ele próprio a raiz -do subvolume ou do volume, o SnapRAID cria um diretório \`.snapraid/\` -diretamente dentro dele (em Btrfs, Bcachefs e NTFS). Embora o SnapRAID -exclua automaticamente esse diretório das suas próprias verificações, -ferramentas externas de cópia de segurança ou de sincronização irão -encontrá\-lo. Se não for excluído, podem atravessar os instantâneos ou -ligações de diretório e copiar toda a árvore de dados congelada, duplicando -dados e esgotando o armazenamento de backup. Configure sempre as -ferramentas de cópia de segurança externas para excluir o diretório -\`.snapraid/\`. -.PP -No ZFS, os instantâneos são expostos através do diretório virtual padrão -\`.zfs/snapshot/\` em vez de \`.snapraid/\`. Por predefinição (\`snapdir=hidden\`), -o ZFS oculta esse diretório das listagens, de modo que ferramentas externas -de cópia de segurança não o percorrem. Se tiver ativado \`snapdir=visible\`, -certifique\-se de que as suas ferramentas de cópia de segurança excluem o -diretório \`.zfs/\`. .SH PADRãO (PATTERN) Os padrões fornecem uma maneira flexível de filtrar arquivos para inclusão ou exclusão. Ao usar caracteres de globbing, você pode definir regras que diff --git a/doc/snapraid-pt.d b/doc/snapraid-pt.d index 029e06b1..e18a4cbd 100644 --- a/doc/snapraid-pt.d +++ b/doc/snapraid-pt.d @@ -1597,37 +1597,6 @@ Ciclo de Vida dos Instantâneos (Snapshots Lifecycle) esse instantâneo, tornando as versões intermediárias dos arquivos indisponíveis para recuperação. - Cópias de segurança externas com instantâneos do sistema de ficheiros (External backups with filesystem snapshots) - Quando os instantâneos do sistema de ficheiros estão ativados, o SnapRAID - acede ou cria instantâneos na raiz de cada subvolume, volume ou conjunto - de dados (dataset). - - Uma forma simples de evitar qualquer interferência com ferramentas externas - é apontar a configuração `data` para um subdiretório normal dentro do - subvolume ou volume (por exemplo, `/mnt/subvol/data` em vez de `/mnt/subvol`). - O SnapRAID cria sempre o diretório de instantâneos na raiz do subvolume - (por exemplo, `/mnt/subvol/.snapraid/`), deixando o subdiretório de dados - limpo, de modo que os instantâneos permaneçam fora e invisíveis para as - ferramentas de cópia de segurança que operam dentro dele. - - Caso contrário, se o diretório `data` configurado for ele próprio a raiz - do subvolume ou do volume, o SnapRAID cria um diretório `.snapraid/` - diretamente dentro dele (em Btrfs, Bcachefs e NTFS). Embora o SnapRAID - exclua automaticamente esse diretório das suas próprias verificações, - ferramentas externas de cópia de segurança ou de sincronização irão - encontrá-lo. Se não for excluído, podem atravessar os instantâneos ou - ligações de diretório e copiar toda a árvore de dados congelada, duplicando - dados e esgotando o armazenamento de backup. Configure sempre as - ferramentas de cópia de segurança externas para excluir o diretório - `.snapraid/`. - - No ZFS, os instantâneos são expostos através do diretório virtual padrão - `.zfs/snapshot/` em vez de `.snapraid/`. Por predefinição (`snapdir=hidden`), - o ZFS oculta esse diretório das listagens, de modo que ferramentas externas - de cópia de segurança não o percorrem. Se tiver ativado `snapdir=visible`, - certifique-se de que as suas ferramentas de cópia de segurança excluem o - diretório `.zfs/`. - Padrão (Pattern) Os padrões fornecem uma maneira flexível de filtrar arquivos para inclusão ou exclusão. Ao usar caracteres de globbing, você pode definir regras que diff --git a/doc/snapraid-pt.txt b/doc/snapraid-pt.txt index 654c4c93..a6d6edb4 100644 --- a/doc/snapraid-pt.txt +++ b/doc/snapraid-pt.txt @@ -1707,39 +1707,6 @@ instantâneo usado pelo sync interrompido. Um sync posterior pode substituir esse instantâneo, tornando as versões intermediárias dos arquivos indisponíveis para recuperação. -9.1 Cópias de segurança externas com instantâneos do sistema de ficheiros (External backups with filesystem snapshots) ----------------------------------------------------------------------------------------------------------------------- - -Quando os instantâneos do sistema de ficheiros estão ativados, o SnapRAID -acede ou cria instantâneos na raiz de cada subvolume, volume ou conjunto -de dados (dataset). - -Uma forma simples de evitar qualquer interferência com ferramentas externas -é apontar a configuração `data` para um subdiretório normal dentro do -subvolume ou volume (por exemplo, `/mnt/subvol/data` em vez de `/mnt/subvol`). -O SnapRAID cria sempre o diretório de instantâneos na raiz do subvolume -(por exemplo, `/mnt/subvol/.snapraid/`), deixando o subdiretório de dados -limpo, de modo que os instantâneos permaneçam fora e invisíveis para as -ferramentas de cópia de segurança que operam dentro dele. - -Caso contrário, se o diretório `data` configurado for ele próprio a raiz -do subvolume ou do volume, o SnapRAID cria um diretório `.snapraid/` -diretamente dentro dele (em Btrfs, Bcachefs e NTFS). Embora o SnapRAID -exclua automaticamente esse diretório das suas próprias verificações, -ferramentas externas de cópia de segurança ou de sincronização irão -encontrá-lo. Se não for excluído, podem atravessar os instantâneos ou -ligações de diretório e copiar toda a árvore de dados congelada, duplicando -dados e esgotando o armazenamento de backup. Configure sempre as -ferramentas de cópia de segurança externas para excluir o diretório -`.snapraid/`. - -No ZFS, os instantâneos são expostos através do diretório virtual padrão -`.zfs/snapshot/` em vez de `.snapraid/`. Por predefinição (`snapdir=hidden`), -o ZFS oculta esse diretório das listagens, de modo que ferramentas externas -de cópia de segurança não o percorrem. Se tiver ativado `snapdir=visible`, -certifique-se de que as suas ferramentas de cópia de segurança excluem o -diretório `.zfs/`. - 10 PADRãO (PATTERN) =================== diff --git a/doc/snapraid-ro.1 b/doc/snapraid-ro.1 index 1d509188..a9036c6f 100644 --- a/doc/snapraid-ro.1 +++ b/doc/snapraid-ro.1 @@ -1952,36 +1952,6 @@ cu succes. Paritatea poate conține în continuare blocuri calculate din snapshot\-ul utilizat de sync\-ul întrerupt. Un sync ulterior poate înlocui acel snapshot, făcând versiunile intermediare ale fișierelor indisponibile pentru recuperare. -.SS Copii de rezervă externe cu snapshot\-uri de sistem de fișiere (External backups with filesystem snapshots) -Când snapshot\-urile sistemului de fișiere sunt activate, SnapRAID accesează -sau creează snapshot\-uri la rădăcina fiecărui subvolum, volum sau set de -date (dataset). -.PP -O modalitate simplă de a evita orice interferență cu instrumentele externe -este de a configura discul \`data\` astfel încât să indice spre un subdirector -obișnuit din cadrul subvolumului sau volumului (de exemplu -\`/mnt/subvol/data\` în loc de \`/mnt/subvol\`). SnapRAID creează întotdeauna -directorul de snapshot\-uri la rădăcina subvolumului (de exemplu -\`/mnt/subvol/.snapraid/\`), lăsând subdirectorul de date curat, astfel încât -snapshot\-urile să rămână în exterior și invizibile pentru instrumentele de -backup care operează în interiorul său. -.PP -În caz contrar, dacă directorul \`data\` configurat este chiar rădăcina -subvolumului sau a volumului, SnapRAID creează un director \`.snapraid/\` -direct în interiorul acestuia (pe Btrfs, Bcachefs și NTFS). Deși SnapRAID -exclude automat acest director din scanarea proprie, instrumentele externe -de backup sau sincronizare îl vor întâlni. Dacă nu este exclus, acestea pot -traversa snapshot\-urile sau legăturile de directoare și pot copia întregul -arbore de date înghețat, duplicând datele și epuizând spațiul de stocare al -backup\-ului. Configurați întotdeauna instrumentele externe de backup să -excludă directorul \`.snapraid/\`. -.PP -Pe ZFS, snapshot\-urile sunt expuse prin directorul virtual standard -\`.zfs/snapshot/\` în loc de \`.snapraid/\`. În mod implicit (\`snapdir=hidden\`), -ZFS ascunde acest director din listările de directoare, astfel încât -instrumentele externe de backup nu îl vor parcurge. Dacă ați activat -\`snapdir=visible\`, asigurați\-vă că instrumentele de backup exclud -directorul \`.zfs/\`. .SH MODEL (PATTERN) Modelele oferă o modalitate flexibilă de a filtra fișierele pentru includere sau excludere. Folosind caractere de tip globbing, puteți defini reguli care să diff --git a/doc/snapraid-ro.d b/doc/snapraid-ro.d index e9d89159..3da8a3a9 100644 --- a/doc/snapraid-ro.d +++ b/doc/snapraid-ro.d @@ -1557,37 +1557,6 @@ Ciclul de viață al snapshot-urilor (Snapshots Lifecycle) acel snapshot, făcând versiunile intermediare ale fișierelor indisponibile pentru recuperare. - Copii de rezervă externe cu snapshot-uri de sistem de fișiere (External backups with filesystem snapshots) - Când snapshot-urile sistemului de fișiere sunt activate, SnapRAID accesează - sau creează snapshot-uri la rădăcina fiecărui subvolum, volum sau set de - date (dataset). - - O modalitate simplă de a evita orice interferență cu instrumentele externe - este de a configura discul `data` astfel încât să indice spre un subdirector - obișnuit din cadrul subvolumului sau volumului (de exemplu - `/mnt/subvol/data` în loc de `/mnt/subvol`). SnapRAID creează întotdeauna - directorul de snapshot-uri la rădăcina subvolumului (de exemplu - `/mnt/subvol/.snapraid/`), lăsând subdirectorul de date curat, astfel încât - snapshot-urile să rămână în exterior și invizibile pentru instrumentele de - backup care operează în interiorul său. - - În caz contrar, dacă directorul `data` configurat este chiar rădăcina - subvolumului sau a volumului, SnapRAID creează un director `.snapraid/` - direct în interiorul acestuia (pe Btrfs, Bcachefs și NTFS). Deși SnapRAID - exclude automat acest director din scanarea proprie, instrumentele externe - de backup sau sincronizare îl vor întâlni. Dacă nu este exclus, acestea pot - traversa snapshot-urile sau legăturile de directoare și pot copia întregul - arbore de date înghețat, duplicând datele și epuizând spațiul de stocare al - backup-ului. Configurați întotdeauna instrumentele externe de backup să - excludă directorul `.snapraid/`. - - Pe ZFS, snapshot-urile sunt expuse prin directorul virtual standard - `.zfs/snapshot/` în loc de `.snapraid/`. În mod implicit (`snapdir=hidden`), - ZFS ascunde acest director din listările de directoare, astfel încât - instrumentele externe de backup nu îl vor parcurge. Dacă ați activat - `snapdir=visible`, asigurați-vă că instrumentele de backup exclud - directorul `.zfs/`. - Model (Pattern) Modelele oferă o modalitate flexibilă de a filtra fișierele pentru includere sau excludere. Folosind caractere de tip globbing, puteți defini reguli care să diff --git a/doc/snapraid-ro.txt b/doc/snapraid-ro.txt index c9d9a2ff..6a3e3880 100644 --- a/doc/snapraid-ro.txt +++ b/doc/snapraid-ro.txt @@ -1667,39 +1667,6 @@ snapshot-ul utilizat de sync-ul întrerupt. Un sync ulterior poate înlocui acel snapshot, făcând versiunile intermediare ale fișierelor indisponibile pentru recuperare. -9.1 Copii de rezervă externe cu snapshot-uri de sistem de fișiere (External backups with filesystem snapshots) --------------------------------------------------------------------------------------------------------------- - -Când snapshot-urile sistemului de fișiere sunt activate, SnapRAID accesează -sau creează snapshot-uri la rădăcina fiecărui subvolum, volum sau set de -date (dataset). - -O modalitate simplă de a evita orice interferență cu instrumentele externe -este de a configura discul `data` astfel încât să indice spre un subdirector -obișnuit din cadrul subvolumului sau volumului (de exemplu -`/mnt/subvol/data` în loc de `/mnt/subvol`). SnapRAID creează întotdeauna -directorul de snapshot-uri la rădăcina subvolumului (de exemplu -`/mnt/subvol/.snapraid/`), lăsând subdirectorul de date curat, astfel încât -snapshot-urile să rămână în exterior și invizibile pentru instrumentele de -backup care operează în interiorul său. - -În caz contrar, dacă directorul `data` configurat este chiar rădăcina -subvolumului sau a volumului, SnapRAID creează un director `.snapraid/` -direct în interiorul acestuia (pe Btrfs, Bcachefs și NTFS). Deși SnapRAID -exclude automat acest director din scanarea proprie, instrumentele externe -de backup sau sincronizare îl vor întâlni. Dacă nu este exclus, acestea pot -traversa snapshot-urile sau legăturile de directoare și pot copia întregul -arbore de date înghețat, duplicând datele și epuizând spațiul de stocare al -backup-ului. Configurați întotdeauna instrumentele externe de backup să -excludă directorul `.snapraid/`. - -Pe ZFS, snapshot-urile sunt expuse prin directorul virtual standard -`.zfs/snapshot/` în loc de `.snapraid/`. În mod implicit (`snapdir=hidden`), -ZFS ascunde acest director din listările de directoare, astfel încât -instrumentele externe de backup nu îl vor parcurge. Dacă ați activat -`snapdir=visible`, asigurați-vă că instrumentele de backup exclud -directorul `.zfs/`. - 10 MODEL (PATTERN) ================== diff --git a/doc/snapraid-ru.1 b/doc/snapraid-ru.1 index 4b5e257f..bcfb9ca0 100644 --- a/doc/snapraid-ru.1 +++ b/doc/snapraid-ru.1 @@ -1957,34 +1957,6 @@ SnapRAID управляет тремя конкретными снимками, использованного прерванной синхронизацией. Более поздняя синхронизация может заменить этот снимок, сделав промежуточные версии файлов недоступными для восстановления. -.SS Внешнее резервное копирование со снимками файловой системы (External backups with filesystem snapshots) -Когда снимки файловой системы включены, SnapRAID обращается к снимкам или -создает их в корне каждого подтома, тома или набора данных (dataset). -.PP -Простой способ избежать любых конфликтов со сторонними инструментами — -указать в директиве \`data\` обычный подкаталог внутри подтома или тома -(например, \`/mnt/subvol/data\` вместо \`/mnt/subvol\`). SnapRAID всегда -создает каталог снимков в корне подтома (например, -\`/mnt/subvol/.snapraid/\`), оставив подкаталог данных чистым, благодаря -чему снимки останутся снаружи и будут невидимы для программ резервного -копирования, работающих внутри него. -.PP -В противном случае, если настроенный каталог \`data\` сам по себе является -корнем подтома или тома, SnapRAID создает каталог \`.snapraid/\` прямо внутри -него (в Btrfs, Bcachefs и NTFS). Хотя SnapRAID автоматически исключает этот -каталог из собственного сканирования, внешние утилиты резервного -копирования или синхронизации файлов обнаружат его. Если каталог не -исключен, они могут войти в снимки или ссылки на каталоги и скопировать -все замороженное дерево данных, дублируя файлы и исчерпывая место в -хранилище резервных копий. Всегда настраивайте внешние утилиты резервного -копирования на исключение каталога \`.snapraid/\`. -.PP -В ZFS снимки доступны через стандартный виртуальный каталог -\`.zfs/snapshot/\`, а не через \`.snapraid/\`. По умолчанию (\`snapdir=hidden\`) -ZFS скрывает этот каталог из списков директорий, поэтому внешние утилиты -резервного копирования не будут сканировать его. Если вы включили -\`snapdir=visible\`, обязательно настройте утилиты резервного копирования на -исключение каталога \`.zfs/\`. .SH Шаблон (PATTERN) Шаблоны обеспечивают гибкий способ фильтрации файлов для включения или исключения. Используя символы подстановки (globbing), вы можете определить правила, diff --git a/doc/snapraid-ru.d b/doc/snapraid-ru.d index 2ce06ad5..c2097a95 100644 --- a/doc/snapraid-ru.d +++ b/doc/snapraid-ru.d @@ -1564,35 +1564,6 @@ Name{number} заменить этот снимок, сделав промежуточные версии файлов недоступными для восстановления. - Внешнее резервное копирование со снимками файловой системы (External backups with filesystem snapshots) - Когда снимки файловой системы включены, SnapRAID обращается к снимкам или - создает их в корне каждого подтома, тома или набора данных (dataset). - - Простой способ избежать любых конфликтов со сторонними инструментами — - указать в директиве `data` обычный подкаталог внутри подтома или тома - (например, `/mnt/subvol/data` вместо `/mnt/subvol`). SnapRAID всегда - создает каталог снимков в корне подтома (например, - `/mnt/subvol/.snapraid/`), оставив подкаталог данных чистым, благодаря - чему снимки останутся снаружи и будут невидимы для программ резервного - копирования, работающих внутри него. - - В противном случае, если настроенный каталог `data` сам по себе является - корнем подтома или тома, SnapRAID создает каталог `.snapraid/` прямо внутри - него (в Btrfs, Bcachefs и NTFS). Хотя SnapRAID автоматически исключает этот - каталог из собственного сканирования, внешние утилиты резервного - копирования или синхронизации файлов обнаружат его. Если каталог не - исключен, они могут войти в снимки или ссылки на каталоги и скопировать - все замороженное дерево данных, дублируя файлы и исчерпывая место в - хранилище резервных копий. Всегда настраивайте внешние утилиты резервного - копирования на исключение каталога `.snapraid/`. - - В ZFS снимки доступны через стандартный виртуальный каталог - `.zfs/snapshot/`, а не через `.snapraid/`. По умолчанию (`snapdir=hidden`) - ZFS скрывает этот каталог из списков директорий, поэтому внешние утилиты - резервного копирования не будут сканировать его. Если вы включили - `snapdir=visible`, обязательно настройте утилиты резервного копирования на - исключение каталога `.zfs/`. - Шаблон (Pattern) Шаблоны обеспечивают гибкий способ фильтрации файлов для включения или исключения. Используя символы подстановки (globbing), вы можете определить правила, diff --git a/doc/snapraid-ru.txt b/doc/snapraid-ru.txt index e1f6a0e6..7133e41b 100644 --- a/doc/snapraid-ru.txt +++ b/doc/snapraid-ru.txt @@ -1674,37 +1674,6 @@ SnapRAID управляет тремя конкретными снимками, заменить этот снимок, сделав промежуточные версии файлов недоступными для восстановления. -9.1 Внешнее резервное копирование со снимками файловой системы (External backups with filesystem snapshots) ------------------------------------------------------------------------------------------------------------ - -Когда снимки файловой системы включены, SnapRAID обращается к снимкам или -создает их в корне каждого подтома, тома или набора данных (dataset). - -Простой способ избежать любых конфликтов со сторонними инструментами — -указать в директиве `data` обычный подкаталог внутри подтома или тома -(например, `/mnt/subvol/data` вместо `/mnt/subvol`). SnapRAID всегда -создает каталог снимков в корне подтома (например, -`/mnt/subvol/.snapraid/`), оставив подкаталог данных чистым, благодаря -чему снимки останутся снаружи и будут невидимы для программ резервного -копирования, работающих внутри него. - -В противном случае, если настроенный каталог `data` сам по себе является -корнем подтома или тома, SnapRAID создает каталог `.snapraid/` прямо внутри -него (в Btrfs, Bcachefs и NTFS). Хотя SnapRAID автоматически исключает этот -каталог из собственного сканирования, внешние утилиты резервного -копирования или синхронизации файлов обнаружат его. Если каталог не -исключен, они могут войти в снимки или ссылки на каталоги и скопировать -все замороженное дерево данных, дублируя файлы и исчерпывая место в -хранилище резервных копий. Всегда настраивайте внешние утилиты резервного -копирования на исключение каталога `.snapraid/`. - -В ZFS снимки доступны через стандартный виртуальный каталог -`.zfs/snapshot/`, а не через `.snapraid/`. По умолчанию (`snapdir=hidden`) -ZFS скрывает этот каталог из списков директорий, поэтому внешние утилиты -резервного копирования не будут сканировать его. Если вы включили -`snapdir=visible`, обязательно настройте утилиты резервного копирования на -исключение каталога `.zfs/`. - 10 Шаблон (PATTERN) =================== diff --git a/doc/snapraid-sv.1 b/doc/snapraid-sv.1 index 769baa66..9173ff54 100644 --- a/doc/snapraid-sv.1 +++ b/doc/snapraid-sv.1 @@ -1953,34 +1953,6 @@ framgångsrikt. Pariteten kan fortfarande innehålla block som beräknats från ögonblicksbilden som användes av den avbrutna synkroniseringen. En senare synkronisering kan ersätta den ögonblicksbilden, vilket gör mellanliggande filversioner otillgängliga för återställning. -.SS Externa säkerhetskopior med ögonblicksbilder av filsystem (External backups with filesystem snapshots) -När ögonblicksbilder av filsystem är aktiverade ansluter eller skapar -SnapRAID ögonblicksbilder vid roten av varje dataundervolym, volym eller -datauppsättning (dataset). -.PP -Ett enkelt sätt att undvika konflikter med externa verktyg är att peka -din \`data\`\-konfiguration till en vanlig underkatalog i undervolymen eller -volymen (till exempel \`/mnt/subvol/data\` istället för \`/mnt/subvol\`). -SnapRAID skapar alltid katalogen för ögonblicksbilder i undervolymens rot -(till exempel \`/mnt/subvol/.snapraid/\`), vilket håller underkatalogen för -data ren så att ögonblicksbilderna förblir utanför och osynliga för -säkerhetskopieringsverktyg som arbetar i den. -.PP -Annars, om din konfigurerade \`data\`\-katalog i sig är roten på undervolymen -eller volymen, skapar SnapRAID en \`.snapraid/\`\-katalog direkt i den (på -Btrfs, Bcachefs och NTFS). Även om SnapRAID automatiskt utesluter denna -katalog från sin egen skanning, kommer externa säkerhetskopieringsverktyg -eller synkroniseringsprogram att stöta på den. Om den inte utesluts kan de -gå in i ögonblicksbilderna eller kataloglänkarna och kopiera hela det frysta -dataträdet, vilket duplicerar data och kan tömma lagringsutrymmet för -säkerhetskopior. Konfigurera alltid externa säkerhetskopieringsverktyg till -att utesluta katalogen \`.snapraid/\`. -.PP -På ZFS exponeras ögonblicksbilder via den virtuella standardkatalogen -\`.zfs/snapshot/\` istället för \`.snapraid/\`. Som standard (\`snapdir=hidden\`) -döljer ZFS denna katalog från kataloglistor så att externa säkerhetskopieringsverktyg -inte söker igenom den. Om du har aktiverat \`snapdir=visible\`, se till att dina -säkerhetskopieringsverktyg utesluter katalogen \`.zfs/\`. .SH MöNSTER (PATTERN) Mönster ger ett flexibelt sätt att filtrera filer för inkludering eller exkludering. Genom att använda jokertecken kan du definiera regler som diff --git a/doc/snapraid-sv.d b/doc/snapraid-sv.d index c49f0796..cba81745 100644 --- a/doc/snapraid-sv.d +++ b/doc/snapraid-sv.d @@ -1558,35 +1558,6 @@ Livscykel för ögonblicksbilder (Snapshots Lifecycle) synkronisering kan ersätta den ögonblicksbilden, vilket gör mellanliggande filversioner otillgängliga för återställning. - Externa säkerhetskopior med ögonblicksbilder av filsystem (External backups with filesystem snapshots) - När ögonblicksbilder av filsystem är aktiverade ansluter eller skapar - SnapRAID ögonblicksbilder vid roten av varje dataundervolym, volym eller - datauppsättning (dataset). - - Ett enkelt sätt att undvika konflikter med externa verktyg är att peka - din `data`-konfiguration till en vanlig underkatalog i undervolymen eller - volymen (till exempel `/mnt/subvol/data` istället för `/mnt/subvol`). - SnapRAID skapar alltid katalogen för ögonblicksbilder i undervolymens rot - (till exempel `/mnt/subvol/.snapraid/`), vilket håller underkatalogen för - data ren så att ögonblicksbilderna förblir utanför och osynliga för - säkerhetskopieringsverktyg som arbetar i den. - - Annars, om din konfigurerade `data`-katalog i sig är roten på undervolymen - eller volymen, skapar SnapRAID en `.snapraid/`-katalog direkt i den (på - Btrfs, Bcachefs och NTFS). Även om SnapRAID automatiskt utesluter denna - katalog från sin egen skanning, kommer externa säkerhetskopieringsverktyg - eller synkroniseringsprogram att stöta på den. Om den inte utesluts kan de - gå in i ögonblicksbilderna eller kataloglänkarna och kopiera hela det frysta - dataträdet, vilket duplicerar data och kan tömma lagringsutrymmet för - säkerhetskopior. Konfigurera alltid externa säkerhetskopieringsverktyg till - att utesluta katalogen `.snapraid/`. - - På ZFS exponeras ögonblicksbilder via den virtuella standardkatalogen - `.zfs/snapshot/` istället för `.snapraid/`. Som standard (`snapdir=hidden`) - döljer ZFS denna katalog från kataloglistor så att externa säkerhetskopieringsverktyg - inte söker igenom den. Om du har aktiverat `snapdir=visible`, se till att dina - säkerhetskopieringsverktyg utesluter katalogen `.zfs/`. - Mönster (Pattern) Mönster ger ett flexibelt sätt att filtrera filer för inkludering eller exkludering. Genom att använda jokertecken kan du definiera regler som diff --git a/doc/snapraid-sv.txt b/doc/snapraid-sv.txt index 59bec6b3..beb8e6a1 100644 --- a/doc/snapraid-sv.txt +++ b/doc/snapraid-sv.txt @@ -1668,37 +1668,6 @@ framgångsrikt. Pariteten kan fortfarande innehålla block som beräknats från synkronisering kan ersätta den ögonblicksbilden, vilket gör mellanliggande filversioner otillgängliga för återställning. -9.1 Externa säkerhetskopior med ögonblicksbilder av filsystem (External backups with filesystem snapshots) ----------------------------------------------------------------------------------------------------------- - -När ögonblicksbilder av filsystem är aktiverade ansluter eller skapar -SnapRAID ögonblicksbilder vid roten av varje dataundervolym, volym eller -datauppsättning (dataset). - -Ett enkelt sätt att undvika konflikter med externa verktyg är att peka -din `data`-konfiguration till en vanlig underkatalog i undervolymen eller -volymen (till exempel `/mnt/subvol/data` istället för `/mnt/subvol`). -SnapRAID skapar alltid katalogen för ögonblicksbilder i undervolymens rot -(till exempel `/mnt/subvol/.snapraid/`), vilket håller underkatalogen för -data ren så att ögonblicksbilderna förblir utanför och osynliga för -säkerhetskopieringsverktyg som arbetar i den. - -Annars, om din konfigurerade `data`-katalog i sig är roten på undervolymen -eller volymen, skapar SnapRAID en `.snapraid/`-katalog direkt i den (på -Btrfs, Bcachefs och NTFS). Även om SnapRAID automatiskt utesluter denna -katalog från sin egen skanning, kommer externa säkerhetskopieringsverktyg -eller synkroniseringsprogram att stöta på den. Om den inte utesluts kan de -gå in i ögonblicksbilderna eller kataloglänkarna och kopiera hela det frysta -dataträdet, vilket duplicerar data och kan tömma lagringsutrymmet för -säkerhetskopior. Konfigurera alltid externa säkerhetskopieringsverktyg till -att utesluta katalogen `.snapraid/`. - -På ZFS exponeras ögonblicksbilder via den virtuella standardkatalogen -`.zfs/snapshot/` istället för `.snapraid/`. Som standard (`snapdir=hidden`) -döljer ZFS denna katalog från kataloglistor så att externa säkerhetskopieringsverktyg -inte söker igenom den. Om du har aktiverat `snapdir=visible`, se till att dina -säkerhetskopieringsverktyg utesluter katalogen `.zfs/`. - 10 MöNSTER (PATTERN) ==================== diff --git a/doc/snapraid-uk.1 b/doc/snapraid-uk.1 index ba98c152..766ceae0 100644 --- a/doc/snapraid-uk.1 +++ b/doc/snapraid-uk.1 @@ -2126,34 +2126,6 @@ SnapRAID керує трьома конкретними знімками, \`stab використаного перерваною синхронізацією. Пізніша синхронізація може замінити цей знімок, зробивши проміжні версії файлів недоступними для відновлення. -.SS Зовнішнє резервне копіювання зі знімками файлової системи (External backups with filesystem snapshots) -Коли знімки файлової системи ввімкнено, SnapRAID звертається до знімків або -створює їх у корені кожного підтому даних, тому або набору даних (dataset). -.PP -Простий спосіб уникнути будь\-яких конфліктів зі сторонніми інструментами — -вказати в налаштуванні \`data\` звичайний підкаталог усередині підтому або -тому (наприклад, \`/mnt/subvol/data\` замість \`/mnt/subvol\`). SnapRAID завжди -створює каталог знімків у корені підтому (наприклад, -\`/mnt/subvol/.snapraid/\`), залишивши підкаталог даних чистим, завдяки чому -знімки залишаться ззовні та будуть невидимими для засобів резервного -копіювання, що працюють у ньому. -.PP -В іншому випадку, якщо налаштований каталог \`data\` сам по собі є коренем -підтому або тому, SnapRAID створює каталог \`.snapraid/\` безпосередньо -всередині нього (у Btrfs, Bcachefs та NTFS). Хоча SnapRAID автоматично -виключає цей каталог із власного сканування, зовнішні інструменти -резервного копіювання або синхронізації файлів виявлять його. Якщо каталог -не виключено, вони можуть увійти у знімки або посилання на каталоги та -скопіювати все заморожене дерево даних, дублюючи файли й вичерпуючи -дисковий простір резервного сховища. Завжди налаштовуйте зовнішні засоби -резервного копіювання на виключення каталогу \`.snapraid/\`. -.PP -У ZFS знімки надаються через стандартний віртуальний каталог -\`.zfs/snapshot/\` замість \`.snapraid/\`. За замовчуванням (\`snapdir=hidden\`) -ZFS приховує цей каталог зі списків каталогів, тому зовнішні інструменти -резервного копіювання не скануватимуть його. Якщо ви ввімкнули -\`snapdir=visible\`, обов\'язково переконайтеся, що ваші інструменти -резервного копіювання виключають каталог \`.zfs/\`. .SH Шаблон (PATTERN) Шаблони забезпечують гнучкий спосіб фільтрації файлів для включення або виключення. Використовуючи символи підстановки (globbing), ви можете визначити правила, які diff --git a/doc/snapraid-uk.d b/doc/snapraid-uk.d index e06dcff0..9e6df76c 100644 --- a/doc/snapraid-uk.d +++ b/doc/snapraid-uk.d @@ -1722,35 +1722,6 @@ Name{number} замінити цей знімок, зробивши проміжні версії файлів недоступними для відновлення. - Зовнішнє резервне копіювання зі знімками файлової системи (External backups with filesystem snapshots) - Коли знімки файлової системи ввімкнено, SnapRAID звертається до знімків або - створює їх у корені кожного підтому даних, тому або набору даних (dataset). - - Простий спосіб уникнути будь-яких конфліктів зі сторонніми інструментами — - вказати в налаштуванні `data` звичайний підкаталог усередині підтому або - тому (наприклад, `/mnt/subvol/data` замість `/mnt/subvol`). SnapRAID завжди - створює каталог знімків у корені підтому (наприклад, - `/mnt/subvol/.snapraid/`), залишивши підкаталог даних чистим, завдяки чому - знімки залишаться ззовні та будуть невидимими для засобів резервного - копіювання, що працюють у ньому. - - В іншому випадку, якщо налаштований каталог `data` сам по собі є коренем - підтому або тому, SnapRAID створює каталог `.snapraid/` безпосередньо - всередині нього (у Btrfs, Bcachefs та NTFS). Хоча SnapRAID автоматично - виключає цей каталог із власного сканування, зовнішні інструменти - резервного копіювання або синхронізації файлів виявлять його. Якщо каталог - не виключено, вони можуть увійти у знімки або посилання на каталоги та - скопіювати все заморожене дерево даних, дублюючи файли й вичерпуючи - дисковий простір резервного сховища. Завжди налаштовуйте зовнішні засоби - резервного копіювання на виключення каталогу `.snapraid/`. - - У ZFS знімки надаються через стандартний віртуальний каталог - `.zfs/snapshot/` замість `.snapraid/`. За замовчуванням (`snapdir=hidden`) - ZFS приховує цей каталог зі списків каталогів, тому зовнішні інструменти - резервного копіювання не скануватимуть його. Якщо ви ввімкнули - `snapdir=visible`, обов'язково переконайтеся, що ваші інструменти - резервного копіювання виключають каталог `.zfs/`. - Шаблон (Pattern) Шаблони забезпечують гнучкий спосіб фільтрації файлів для включення або виключення. Використовуючи символи підстановки (globbing), ви можете визначити правила, які diff --git a/doc/snapraid-uk.txt b/doc/snapraid-uk.txt index 1dcdc967..95d034d8 100644 --- a/doc/snapraid-uk.txt +++ b/doc/snapraid-uk.txt @@ -1832,37 +1832,6 @@ SnapRAID керує трьома конкретними знімками, `stabl замінити цей знімок, зробивши проміжні версії файлів недоступними для відновлення. -9.1 Зовнішнє резервне копіювання зі знімками файлової системи (External backups with filesystem snapshots) ----------------------------------------------------------------------------------------------------------- - -Коли знімки файлової системи ввімкнено, SnapRAID звертається до знімків або -створює їх у корені кожного підтому даних, тому або набору даних (dataset). - -Простий спосіб уникнути будь-яких конфліктів зі сторонніми інструментами — -вказати в налаштуванні `data` звичайний підкаталог усередині підтому або -тому (наприклад, `/mnt/subvol/data` замість `/mnt/subvol`). SnapRAID завжди -створює каталог знімків у корені підтому (наприклад, -`/mnt/subvol/.snapraid/`), залишивши підкаталог даних чистим, завдяки чому -знімки залишаться ззовні та будуть невидимими для засобів резервного -копіювання, що працюють у ньому. - -В іншому випадку, якщо налаштований каталог `data` сам по собі є коренем -підтому або тому, SnapRAID створює каталог `.snapraid/` безпосередньо -всередині нього (у Btrfs, Bcachefs та NTFS). Хоча SnapRAID автоматично -виключає цей каталог із власного сканування, зовнішні інструменти -резервного копіювання або синхронізації файлів виявлять його. Якщо каталог -не виключено, вони можуть увійти у знімки або посилання на каталоги та -скопіювати все заморожене дерево даних, дублюючи файли й вичерпуючи -дисковий простір резервного сховища. Завжди налаштовуйте зовнішні засоби -резервного копіювання на виключення каталогу `.snapraid/`. - -У ZFS знімки надаються через стандартний віртуальний каталог -`.zfs/snapshot/` замість `.snapraid/`. За замовчуванням (`snapdir=hidden`) -ZFS приховує цей каталог зі списків каталогів, тому зовнішні інструменти -резервного копіювання не скануватимуть його. Якщо ви ввімкнули -`snapdir=visible`, обов'язково переконайтеся, що ваші інструменти -резервного копіювання виключають каталог `.zfs/`. - 10 Шаблон (PATTERN) =================== diff --git a/doc/snapraid-zh.1 b/doc/snapraid-zh.1 index c39b06ed..1359424b 100644 --- a/doc/snapraid-zh.1 +++ b/doc/snapraid-zh.1 @@ -1755,25 +1755,6 @@ SnapRAID 在每个数据子卷根目录下的隐藏目录中管理三个特定 但是,如果 \`sync\` 中断,请避免在后续 \`sync\` 成功完成之前修改、删除、 替换或移动文件。校验可能仍包含根据中断的同步所使用的快照计算的块。 以后的同步可能会替换该快照,从而导致中间文件版本无法用于恢复。 -.SS 使用文件系统快照进行外部备份 (External backups with filesystem snapshots) -当启用文件系统快照时,SnapRAID 会在每个数据子卷、卷或数据集(dataset) -的根目录下访问或创建快照。 -.PP -避免与外部工具发生冲突的一种简单方法,是将 \`data\` 配置指向子卷或卷内的 -常规子目录(例如 \`/mnt/subvol/data\` 而不是 \`/mnt/subvol\`)。SnapRAID -始终会在子卷根目录下创建快照目录(例如 \`/mnt/subvol/.snapraid/\`),从而保持 -数据子目录整洁,使快照保留在外部且对在该目录内运行的备份工具不可见。 -.PP -否则,如果配置的 \`data\` 目录本身就是子卷或卷的根目录,SnapRAID 会直接在其中 -创建 \`.snapraid/\` 目录(在 Btrfs、Bcachefs 和 NTFS 上)。虽然 SnapRAID -会在其自身的扫描中自动排除此目录,但外部备份工具或文件同步工具会发现它。如果未 -将其排除,它们可能会遍历快照或目录链接并复制整个冻结的数据树,从而导致数据 -重复并耗尽备份存储空间。请务必将外部备份工具配置为排除 \`.snapraid/\` 目录。 -.PP -在 ZFS 上,快照是通过标准虚拟目录 \`.zfs/snapshot/\` 而不是 \`.snapraid/\` -公开的。默认情况下(\`snapdir=hidden\`),ZFS 会在目录列表中隐藏此目录,因此 -外部备份工具不会遍历它。如果您启用了 \`snapdir=visible\`,请确保您的备份工具 -排除 \`.zfs/\` 目录。 .SH 模式 (PATTERN) 模式提供了一种灵活的方法来过滤要包含或排除的文件。通过使用通配符(globbing), 您可以定义匹配特定文件名或整个目录结构的规则,而无需手动列出每个路径。 diff --git a/doc/snapraid-zh.d b/doc/snapraid-zh.d index f2b0836e..324c5511 100644 --- a/doc/snapraid-zh.d +++ b/doc/snapraid-zh.d @@ -1360,26 +1360,6 @@ Name{number} 替换或移动文件。校验可能仍包含根据中断的同步所使用的快照计算的块。 以后的同步可能会替换该快照,从而导致中间文件版本无法用于恢复。 - 使用文件系统快照进行外部备份 (External backups with filesystem snapshots) - 当启用文件系统快照时,SnapRAID 会在每个数据子卷、卷或数据集(dataset) - 的根目录下访问或创建快照。 - - 避免与外部工具发生冲突的一种简单方法,是将 `data` 配置指向子卷或卷内的 - 常规子目录(例如 `/mnt/subvol/data` 而不是 `/mnt/subvol`)。SnapRAID - 始终会在子卷根目录下创建快照目录(例如 `/mnt/subvol/.snapraid/`),从而保持 - 数据子目录整洁,使快照保留在外部且对在该目录内运行的备份工具不可见。 - - 否则,如果配置的 `data` 目录本身就是子卷或卷的根目录,SnapRAID 会直接在其中 - 创建 `.snapraid/` 目录(在 Btrfs、Bcachefs 和 NTFS 上)。虽然 SnapRAID - 会在其自身的扫描中自动排除此目录,但外部备份工具或文件同步工具会发现它。如果未 - 将其排除,它们可能会遍历快照或目录链接并复制整个冻结的数据树,从而导致数据 - 重复并耗尽备份存储空间。请务必将外部备份工具配置为排除 `.snapraid/` 目录。 - - 在 ZFS 上,快照是通过标准虚拟目录 `.zfs/snapshot/` 而不是 `.snapraid/` - 公开的。默认情况下(`snapdir=hidden`),ZFS 会在目录列表中隐藏此目录,因此 - 外部备份工具不会遍历它。如果您启用了 `snapdir=visible`,请确保您的备份工具 - 排除 `.zfs/` 目录。 - 模式 (Pattern) 模式提供了一种灵活的方法来过滤要包含或排除的文件。通过使用通配符(globbing), 您可以定义匹配特定文件名或整个目录结构的规则,而无需手动列出每个路径。 diff --git a/doc/snapraid-zh.txt b/doc/snapraid-zh.txt index 4791ecd4..76a04230 100644 --- a/doc/snapraid-zh.txt +++ b/doc/snapraid-zh.txt @@ -1470,28 +1470,6 @@ SnapRAID 在每个数据子卷根目录下的隐藏目录中管理三个特定 替换或移动文件。校验可能仍包含根据中断的同步所使用的快照计算的块。 以后的同步可能会替换该快照,从而导致中间文件版本无法用于恢复。 -9.1 使用文件系统快照进行外部备份 (External backups with filesystem snapshots) ---------------------------------------------------------------- - -当启用文件系统快照时,SnapRAID 会在每个数据子卷、卷或数据集(dataset) -的根目录下访问或创建快照。 - -避免与外部工具发生冲突的一种简单方法,是将 `data` 配置指向子卷或卷内的 -常规子目录(例如 `/mnt/subvol/data` 而不是 `/mnt/subvol`)。SnapRAID -始终会在子卷根目录下创建快照目录(例如 `/mnt/subvol/.snapraid/`),从而保持 -数据子目录整洁,使快照保留在外部且对在该目录内运行的备份工具不可见。 - -否则,如果配置的 `data` 目录本身就是子卷或卷的根目录,SnapRAID 会直接在其中 -创建 `.snapraid/` 目录(在 Btrfs、Bcachefs 和 NTFS 上)。虽然 SnapRAID -会在其自身的扫描中自动排除此目录,但外部备份工具或文件同步工具会发现它。如果未 -将其排除,它们可能会遍历快照或目录链接并复制整个冻结的数据树,从而导致数据 -重复并耗尽备份存储空间。请务必将外部备份工具配置为排除 `.snapraid/` 目录。 - -在 ZFS 上,快照是通过标准虚拟目录 `.zfs/snapshot/` 而不是 `.snapraid/` -公开的。默认情况下(`snapdir=hidden`),ZFS 会在目录列表中隐藏此目录,因此 -外部备份工具不会遍历它。如果您启用了 `snapdir=visible`,请确保您的备份工具 -排除 `.zfs/` 目录。 - 10 模式 (PATTERN) =============== diff --git a/doc/snapraid.1 b/doc/snapraid.1 index 6bd30dce..e361f5f9 100644 --- a/doc/snapraid.1 +++ b/doc/snapraid.1 @@ -1962,33 +1962,6 @@ successfully. Parity may still contain blocks computed from the snapshot used by the interrupted sync. A later sync may replace that snapshot, making intermediate file versions unavailable for recovery. -.SS External backups with filesystem snapshots -When filesystem snapshots are enabled, SnapRAID accesses or creates -snapshots at the root of each data subvolume, volume, or dataset. -.PP -A simple way to avoid any interference with external tools is to point -your \`data\` configuration to a regular subdirectory within the subvolume -or volume (for example, \`/mnt/subvol/data\` instead of \`/mnt/subvol\`). -SnapRAID always creates the snapshot directory at the subvolume root -(for example, \`/mnt/subvol/.snapraid/\`), leaving the data subdirectory -clean so that snapshots remain outside and invisible to backup tools -operating inside it. -.PP -Otherwise, if your configured \`data\` directory is itself the subvolume -or volume root, SnapRAID creates the \`.snapraid/\` directory directly -inside it (on Btrfs, Bcachefs, and NTFS). While SnapRAID automatically -excludes this directory from its own scanning, external backup tools -or file synchronizers will encounter it. If not excluded, they may -traverse into the snapshots or directory links and copy the entire -frozen data tree, duplicating data and potentially exhausting backup -storage. Always configure external backup tools to exclude the -\`.snapraid/\` directory. -.PP -On ZFS, snapshots are exposed through the standard \`.zfs/snapshot/\` -virtual directory rather than \`.snapraid/\`. By default (\`snapdir=hidden\`), -ZFS hides this directory from directory listings, so external backup -tools will not traverse it. If you have enabled \`snapdir=visible\`, -ensure your backup tools exclude the \`.zfs/\` directory. .SH PATTERN Patterns are used to select a subset of files to exclude or include in the process. Globbing characters can be used to match files and paths diff --git a/doc/snapraid.d b/doc/snapraid.d index 2682257e..920ac71d 100644 --- a/doc/snapraid.d +++ b/doc/snapraid.d @@ -1567,34 +1567,6 @@ Snapshots Lifecycle that snapshot, making intermediate file versions unavailable for recovery. - External backups with filesystem snapshots - When filesystem snapshots are enabled, SnapRAID accesses or creates - snapshots at the root of each data subvolume, volume, or dataset. - - A simple way to avoid any interference with external tools is to point - your `data` configuration to a regular subdirectory within the subvolume - or volume (for example, `/mnt/subvol/data` instead of `/mnt/subvol`). - SnapRAID always creates the snapshot directory at the subvolume root - (for example, `/mnt/subvol/.snapraid/`), leaving the data subdirectory - clean so that snapshots remain outside and invisible to backup tools - operating inside it. - - Otherwise, if your configured `data` directory is itself the subvolume - or volume root, SnapRAID creates the `.snapraid/` directory directly - inside it (on Btrfs, Bcachefs, and NTFS). While SnapRAID automatically - excludes this directory from its own scanning, external backup tools - or file synchronizers will encounter it. If not excluded, they may - traverse into the snapshots or directory links and copy the entire - frozen data tree, duplicating data and potentially exhausting backup - storage. Always configure external backup tools to exclude the - `.snapraid/` directory. - - On ZFS, snapshots are exposed through the standard `.zfs/snapshot/` - virtual directory rather than `.snapraid/`. By default (`snapdir=hidden`), - ZFS hides this directory from directory listings, so external backup - tools will not traverse it. If you have enabled `snapdir=visible`, - ensure your backup tools exclude the `.zfs/` directory. - Pattern Patterns are used to select a subset of files to exclude or include in the process. Globbing characters can be used to match files and paths diff --git a/doc/snapraid.txt b/doc/snapraid.txt index 6833e6f0..4f425fd3 100644 --- a/doc/snapraid.txt +++ b/doc/snapraid.txt @@ -1677,36 +1677,6 @@ snapshot used by the interrupted sync. A later sync may replace that snapshot, making intermediate file versions unavailable for recovery. -9.1 External backups with filesystem snapshots ----------------------------------------------- - -When filesystem snapshots are enabled, SnapRAID accesses or creates -snapshots at the root of each data subvolume, volume, or dataset. - -A simple way to avoid any interference with external tools is to point -your `data` configuration to a regular subdirectory within the subvolume -or volume (for example, `/mnt/subvol/data` instead of `/mnt/subvol`). -SnapRAID always creates the snapshot directory at the subvolume root -(for example, `/mnt/subvol/.snapraid/`), leaving the data subdirectory -clean so that snapshots remain outside and invisible to backup tools -operating inside it. - -Otherwise, if your configured `data` directory is itself the subvolume -or volume root, SnapRAID creates the `.snapraid/` directory directly -inside it (on Btrfs, Bcachefs, and NTFS). While SnapRAID automatically -excludes this directory from its own scanning, external backup tools -or file synchronizers will encounter it. If not excluded, they may -traverse into the snapshots or directory links and copy the entire -frozen data tree, duplicating data and potentially exhausting backup -storage. Always configure external backup tools to exclude the -`.snapraid/` directory. - -On ZFS, snapshots are exposed through the standard `.zfs/snapshot/` -virtual directory rather than `.snapraid/`. By default (`snapdir=hidden`), -ZFS hides this directory from directory listings, so external backup -tools will not traverse it. If you have enabled `snapdir=visible`, -ensure your backup tools exclude the `.zfs/` directory. - 10 PATTERN ========== diff --git a/os/mingw.c b/os/mingw.c index 955fbfd5..e41d15f5 100644 --- a/os/mingw.c +++ b/os/mingw.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifdef __MINGW32__ /* only for MingW */ +#ifdef __MINGW32__ /* Only for MingW */ #include "os.h" @@ -14,9 +14,9 @@ /****************************************************************************/ /* global */ -/* add missing Windows declaration */ +/* Add missing Windows declaration */ -/* for SetThreadExecutionState */ +/* For SetThreadExecutionState */ #ifndef WIN32_ES_SYSTEM_REQUIRED #define WIN32_ES_SYSTEM_REQUIRED 0x00000001L #endif @@ -33,7 +33,7 @@ #define WIN32_ES_CONTINUOUS 0x80000000L #endif -/* for SetSearchPathMode */ +/* For SetSearchPathMode */ #ifndef BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE #define BASE_SEARCH_PATH_ENABLE_SAFE_SEARCHMODE 0x00000001 #endif @@ -41,7 +41,7 @@ #define BASE_SEARCH_PATH_PERMANENT 0x00008000 #endif -/* file Index */ +/* File Index */ #undef FILE_INVALID_FILE_ID #define FILE_INVALID_FILE_ID ((ULONGLONG)-1LL) #undef FILE_UNSUPPORTED_FILE_ID @@ -64,7 +64,6 @@ static windows_key_t last_error; */ static windows_mutex_t tick_lock; static uint64_t tick_last; -static uint64_t tick_frequency; /** * Mutex for process reference publication, termination, and unpublication. @@ -571,17 +570,17 @@ static BOOL GetReparseTagInfoByHandle(HANDLE hFile, FILE_ATTRIBUTE_TAG_INFO* lpF */ static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct windows_stat* st) { - /* convert special attributes */ + /* Convert special attributes */ if ((FileAttributes & FILE_ATTRIBUTE_DEVICE) != 0) { st->st_mode = S_IFBLK; st->st_desc = "device"; - } else if ((FileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* offline */ + } else if ((FileAttributes & FILE_ATTRIBUTE_OFFLINE) != 0) { /* Offline */ st->st_mode = S_IFCHR; st->st_desc = "offline"; - } else if ((FileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* temporary */ + } else if ((FileAttributes & FILE_ATTRIBUTE_TEMPORARY) != 0) { /* Temporary */ st->st_mode = S_IFCHR; st->st_desc = "temporary"; - } else if ((FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* reparse point */ + } else if ((FileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) != 0) { /* Reparse point */ switch (ReparseTag) { /* if we don't have the ReparseTag information */ case 0 : @@ -622,7 +621,7 @@ static void windows_attr2stat(DWORD FileAttributes, DWORD ReparseTag, struct win st->st_desc = "reparse-point"; break; } - } else if ((FileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* system */ + } else if ((FileAttributes & FILE_ATTRIBUTE_SYSTEM) != 0) { /* System */ if ((FileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0) { st->st_mode = S_IFCHR; st->st_desc = "system-directory"; @@ -696,7 +695,7 @@ static int windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, const FILE_ /* GetFileInformationByHandle() ensures to return synced information */ st->st_sync = 1; - /* + /** * In ReFS the IDs are 128 bit, and the 64 bit interface may fail. * * From Microsoft "Application Compatibility with ReFS" @@ -799,13 +798,13 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s windows_time2stat(mtime, st); - /* no inode information available */ + /* No inode information available */ st->st_ino = INODE_INVALID; - /* no link information available */ + /* No link information available */ st->st_nlink = 0; - /* no device information available */ + /* No device information available */ st->st_dev = 0; /* directory listing doesn't ensure to return synced information */ @@ -878,9 +877,6 @@ void windows_errno(DWORD error) case ERROR_PATH_NOT_FOUND : /* in GetFileAttributeW() if internal path not found */ errno = ENOENT; break; - case ERROR_DIRECTORY : - errno = ENOTDIR; - break; case ERROR_ACCESS_DENIED : /* in CreateDirectoryW() if dir is scheduled for deletion */ case ERROR_CURRENT_DIRECTORY : /* in RemoveDirectoryW() if removing the current directory */ case ERROR_SHARING_VIOLATION : /* in RemoveDirectoryW() if in use */ @@ -1192,8 +1188,8 @@ int windows_fsync(int fd) * So, we use now an extended error reporting. */ windows_errno(error); - os_syslog(OS_LVL_ERROR, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); - os_syslog(OS_LVL_ERROR, "are you using ATA-over-Ethernet? Please report it."); + os_syslog(OS_LVL_INFO, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); + os_syslog(OS_LVL_INFO, "are you using ATA-over-Ethernet? Please report it."); return -1; case ERROR_ACCESS_DENIED : @@ -1249,8 +1245,8 @@ static int windows_vsync(const wchar_t* volume) * So, we use now an extended error reporting. */ windows_errno(error); - os_syslog(OS_LVL_ERROR, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); - os_syslog(OS_LVL_ERROR, "are you using ATA-over-Ethernet? Please report it."); + os_syslog(OS_LVL_INFO, "unexpected Windows INVALID_HANDLE error in FlushFileBuffers()"); + os_syslog(OS_LVL_INFO, "are you using ATA-over-Ethernet? Please report it."); return -1; case ERROR_ACCESS_DENIED : @@ -2079,7 +2075,7 @@ int windows_symlink_directory(const char* existing, const char* file) return windows_symlink_flags(existing, file, SYMBOLIC_LINK_FLAG_DIRECTORY); } -/* adds missing definitions in MingW winnt.h */ +/* Adds missing definitions in MingW winnt.h */ #ifndef FSCTL_GET_REPARSE_POINT #define FSCTL_GET_REPARSE_POINT 0x000900a8 #endif @@ -2258,7 +2254,7 @@ ssize_t windows_read(int fd, void* buffer, size_t size) if (!ReadFile(h, buffer, size, &count, 0)) { DWORD error = GetLastError(); - /* a closed Windows pipe has the same read semantics as a POSIX pipe at EOF. */ + /* A closed Windows pipe has the same read semantics as a POSIX pipe at EOF. */ if (error == ERROR_BROKEN_PIPE) return 0; @@ -2982,7 +2978,7 @@ static int fixcat(WCHAR* cmd, int size, int pos, const WCHAR* arg) return pos; } -/** +/* * Append one argument using the Microsoft C runtime command-line * quoting rules. Use this when building a normal CreateProcessW() * command line that the target process will parse back into argv[]. @@ -3043,7 +3039,7 @@ static int argcat(WCHAR* cmd, int size, int pos, const WCHAR* arg) return pos; } -/** +/* * Append one argument for a batch script invoked through cmd.exe /c. * Unlike argcat(), this must preserve the argument boundary while * preventing cmd.exe command metacharacters from changing the command @@ -3206,14 +3202,14 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* create a pipe for the child process's STDOUT */ if (!CreatePipe(&stdout_read_handle, &stdout_write_handle, &sa, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); return -1; } /* ensure the reading handle to the pipe is not inherited */ if (!SetHandleInformation(stdout_read_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to set handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdout_write_handle); CloseHandle(stdout_read_handle); return -1; @@ -3222,7 +3218,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch out_f = _open_osfhandle((intptr_t)stdout_read_handle, O_RDONLY | O_BINARY); if (out_f == -1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdout_write_handle); CloseHandle(stdout_read_handle); return -1; @@ -3233,7 +3229,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* create a pipe for the child process's STDERR */ if (!CreatePipe(&stderr_read_handle, &stderr_write_handle, &sa, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create pipe for spawn, errno=%s(%d)", strerror(errno), errno); if (has_out) { CloseHandle(stdout_write_handle); close(out_f); @@ -3244,7 +3240,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* ensure the reading handle to the pipe is not inherited */ if (!SetHandleInformation(stderr_read_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to set handle information for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stderr_write_handle); CloseHandle(stderr_read_handle); if (has_out) { @@ -3257,7 +3253,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch err_f = _open_osfhandle((intptr_t)stderr_read_handle, O_RDONLY | O_BINARY); if (err_f == -1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open osfhandle for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stderr_write_handle); CloseHandle(stderr_read_handle); if (has_out) { @@ -3272,7 +3268,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch HANDLE nul_handle = CreateFileW(L"NUL", GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, NULL); if (nul_handle == INVALID_HANDLE_VALUE) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to open NUL for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open NUL for spawn, errno=%s(%d)", strerror(errno), errno); if (has_out) { CloseHandle(stdout_write_handle); close(out_f); @@ -3302,7 +3298,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch } pos = argcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { - os_syslog(OS_LVL_ERROR, "command too long for spawn"); + os_syslog(OS_LVL_INFO, "command to long for spawn"); CloseHandle(nul_handle); if (has_out) { CloseHandle(stdout_write_handle); @@ -3322,7 +3318,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch if (process_startup_init(&startup, nul_handle, has_out ? stdout_write_handle : nul_handle, has_err ? stderr_write_handle : nul_handle) != 0) { - os_syslog(OS_LVL_ERROR, "failed to create handle list for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create handle list for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(nul_handle); if (has_out) { CloseHandle(stdout_write_handle); @@ -3369,12 +3365,12 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch if (!ret) create_error = GetLastError(); } else { - /* drop to restricted service account */ + /* Drop to restricted service account */ HANDLE h_token = NULL; - /* validate that the requested user is actually a supported Service Account before attempting logon */ + /* Validate that the requested user is actually a supported Service Account before attempting logon */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { - os_syslog(OS_LVL_ERROR, "only supported users are LocalService and NetworkService"); + os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(nul_handle); @@ -3392,7 +3388,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch const wchar_t* wuser = _stricmp(run_as_user, "LocalService") == 0 ? L"LocalService" : L"NetworkService"; if (!LogonUserW(wuser, L"NT AUTHORITY", NULL, LOGON32_LOGON_SERVICE, LOGON32_PROVIDER_DEFAULT, &h_token)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(nul_handle); @@ -3407,11 +3403,11 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch return -1; } - /* create an environment block to ensure PATH is loaded */ + /* Create an environment block to ensure PATH is loaded */ LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(h_token); process_startup_done(&startup); @@ -3447,7 +3443,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch process_startup_done(&startup); if (!ret) { windows_errno(create_error); - os_syslog(OS_LVL_ERROR, "failed to create process '%s' for spawn, errno=%s(%d)", argv[0], strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create process '%s' for spawn, errno=%s(%d)", argv[0], strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(nul_handle); if (has_out) { @@ -3471,7 +3467,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch if (os_job_handle != 0) { if (!AssignProcessToJobObject(os_job_handle, pi.hProcess)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to assign process to job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to assign process to job object, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3491,7 +3487,7 @@ pid_t os_spawn(char** argv, int* stdout_read_int, int* stderr_read_int, const ch /* resume the primary thread now that the process is assigned to the job object */ if (ResumeThread(pi.hThread) == (DWORD)-1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to resume thread for spawn, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to resume thread for spawn, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3696,14 +3692,14 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* create pipe for child's STDIN */ if (!CreatePipe(&stdin_read_handle, &stdin_write_handle, &sa, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to create pipe for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create pipe for command, errno=%s(%d)", strerror(errno), errno); return -1; } /* ensure the parent's write end is NOT inherited */ if (!SetHandleInformation(stdin_write_handle, HANDLE_FLAG_INHERIT, 0)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to set handle information for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to handle information for spawn, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); return -1; @@ -3720,7 +3716,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t NULL); if (nul == INVALID_HANDLE_VALUE) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to create nul device, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create nul device, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); return -1; @@ -3728,7 +3724,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t ZeroMemory(&pi, sizeof(pi)); if (process_startup_init(&startup, stdin_read_handle, nul, nul) != 0) { - os_syslog(OS_LVL_ERROR, "failed to create handle list for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create handle list for command, errno=%s(%d)", strerror(errno), errno); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); CloseHandle(nul); @@ -3777,7 +3773,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t * Service Account before attempting logon. */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { - os_syslog(OS_LVL_ERROR, "only supported users are LocalService and NetworkService"); + os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(stdin_read_handle); @@ -3789,7 +3785,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t const wchar_t* wuser = _stricmp(run_as_user, "LocalService") == 0 ? L"LocalService" : L"NetworkService"; if (!LogonUserW(wuser, L"NT AUTHORITY", NULL, LOGON32_LOGON_SERVICE, LOGON32_PROVIDER_DEFAULT, &h_token)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(stdin_read_handle); @@ -3804,7 +3800,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); process_startup_done(&startup); CloseHandle(stdin_read_handle); @@ -3834,7 +3830,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t process_startup_done(&startup); if (!ret) { windows_errno(create_error); - os_syslog(OS_LVL_ERROR, "failed to create process '%s' for command, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create process '%s' for command, errno=%s(%d)", command, strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(stdin_read_handle); CloseHandle(stdin_write_handle); @@ -3852,7 +3848,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (os_job_handle != 0) { if (!AssignProcessToJobObject(os_job_handle, pi.hProcess)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to assign command process to job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to assign command process to job object, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3865,7 +3861,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* resume the primary thread now that the process is assigned to the job object */ if (ResumeThread(pi.hThread) == (DWORD)-1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to resume thread for command, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to resume thread for command, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -3888,7 +3884,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (error == ERROR_BROKEN_PIPE || error == ERROR_NO_DATA) break; /* child closed stdin early */ windows_errno(error); - os_syslog(OS_LVL_ERROR, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(errno), errno); break; } written += count; @@ -3914,7 +3910,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_ERROR, "command %s timeout after %" PRId64 " seconds", command, execution_time); + os_syslog(OS_LVL_INFO, "command %s timeout after %" PRId64 " seconds", command, execution_time); return -1; } @@ -3926,16 +3922,16 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (exit_code == 0) os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with success", command, execution_time); else - os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); + os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Windows it can happen */ - os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", command, execution_time, (unsigned)status); + os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", command, execution_time, (unsigned)status); return -1; } } @@ -4094,7 +4090,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou /* resolve the script path to prevent symlink attacks */ if (!realpath(script_path, resolved_path)) { - os_syslog(OS_LVL_ERROR, "failed to resolve script, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to resolve script, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); return -1; } @@ -4120,9 +4116,9 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou pos = scriptcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { if (errno == EINVAL) - os_syslog(OS_LVL_ERROR, "unsupported embedded double quote in script path"); + os_syslog(OS_LVL_INFO, "unsupported embedded double quote in script path"); else - os_syslog(OS_LVL_ERROR, "command too long for script"); + os_syslog(OS_LVL_INFO, "command to long for script"); return -1; } for (int i = 1; argv[i]; ++i) { @@ -4131,15 +4127,15 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou pos = scriptcat(cmd_buffer, COMMAND_LINE_MAX, pos, conv); if (pos < 0) { if (errno == EINVAL) - os_syslog(OS_LVL_ERROR, "unsupported embedded double quote in script argument"); + os_syslog(OS_LVL_INFO, "unsupported embedded double quote in script argument"); else - os_syslog(OS_LVL_ERROR, "command too long for script"); + os_syslog(OS_LVL_INFO, "command to long for script"); return -1; } } pos = fixcat(cmd_buffer, COMMAND_LINE_MAX, pos, L" \""); if (pos < 0) { - os_syslog(OS_LVL_ERROR, "command too long for script"); + os_syslog(OS_LVL_INFO, "command to long for script"); return -1; } cmd_buffer[pos] = 0; @@ -4167,14 +4163,14 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou base_env = GetEnvironmentStringsW(); if (!base_env) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to get environment strings, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to get environment strings, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } combined_env = env_combine(base_env, envp); FreeEnvironmentStringsW(base_env); if (!combined_env) { - os_syslog(OS_LVL_ERROR, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } @@ -4205,7 +4201,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou * Service Account before attempting logon. */ if (_stricmp(run_as_user, "LocalService") != 0 && _stricmp(run_as_user, "NetworkService") != 0) { - os_syslog(OS_LVL_ERROR, "only supported users are LocalService and NetworkService"); + os_syslog(OS_LVL_INFO, "only supported users are LocalService and NetworkService"); pid_unpublish(pid_slot, 0); return -1; } @@ -4213,7 +4209,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou const wchar_t* wuser = _stricmp(run_as_user, "LocalService") == 0 ? L"LocalService" : L"NetworkService"; if (!LogonUserW(wuser, L"NT AUTHORITY", NULL, LOGON32_LOGON_SERVICE, LOGON32_PROVIDER_DEFAULT, &h_token)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to logon user %s, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } @@ -4224,7 +4220,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou LPVOID env = NULL; if (!CreateEnvironmentBlock(&env, h_token, FALSE)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to get user %s environment, errno=%s(%d)", run_as_user, strerror(errno), errno); pid_unpublish(pid_slot, 0); CloseHandle(h_token); return -1; @@ -4234,7 +4230,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (envp != NULL) { combined_env = env_combine((const WCHAR*)env, envp); if (!combined_env) { - os_syslog(OS_LVL_ERROR, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to combine environment strings, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); DestroyEnvironmentBlock(env); CloseHandle(h_token); @@ -4261,7 +4257,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou } if (!ret) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to create process '%s' for script, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create process '%s' for script, errno=%s(%d)", resolved_path, strerror(errno), errno); pid_unpublish(pid_slot, 0); return -1; } @@ -4270,7 +4266,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (os_job_handle != 0) { if (!AssignProcessToJobObject(os_job_handle, pi.hProcess)) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to assign script process to job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to assign script process to job object, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -4282,7 +4278,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou /* resume the primary thread now that the process is assigned to the job object */ if (ResumeThread(pi.hThread) == (DWORD)-1) { windows_errno(GetLastError()); - os_syslog(OS_LVL_ERROR, "failed to resume thread for script, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to resume thread for script, errno=%s(%d)", strerror(errno), errno); pid_unpublish(pid_slot, 0); TerminateProcess(pi.hProcess, 1); CloseHandle(pi.hThread); @@ -4309,7 +4305,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_ERROR, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); + os_syslog(OS_LVL_INFO, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); return -1; } @@ -4321,16 +4317,16 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (exit_code == 0) os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with success", resolved_path, execution_time); else - os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); + os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Windows it can happen */ - os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", resolved_path, execution_time, (unsigned)status); + os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds for unknown reason, status=0x%08x", resolved_path, execution_time, (unsigned)status); return -1; } } @@ -4385,7 +4381,7 @@ OS_FILE* os_popen(const char** argv) fail_child: saved_errno = errno; close(stdout_fd); - /* terminate and reap the child immediately so no process or handle is leaked on stream creation failure. */ + /* Terminate and reap the child immediately so no process or handle is leaked on stream creation failure. */ pid_kill(pid); status = 0; os_wait(pid, &status, 0); @@ -4409,15 +4405,15 @@ int os_pclose(OS_FILE* stream) int close_ret; pid_t wait_ret; - /* keep the owned resources locally so the OS_FILE container can be released immediately. */ + /* Keep the owned resources locally so the OS_FILE container can be released immediately. */ free(stream); - /* closing the stream also closes the CRT descriptor and the underlying stdout pipe handle. */ + /* Closing the stream also closes the CRT descriptor and the underlying stdout pipe handle. */ close_ret = fclose(fp); if (close_ret != 0) saved_errno = errno; - /* always reap the child even if closing the stream failed. */ + /* Always reap the child even if closing the stream failed. */ wait_ret = os_wait(pid, &status, 0); if (wait_ret < 0 && saved_errno == 0) saved_errno = errno; @@ -4499,24 +4495,6 @@ uint64_t os_tick_ms(void) return GetTickCount64(); } -uint64_t os_tick_us(void) -{ - uint64_t ticks = os_tick(); - uint64_t sec = ticks / tick_frequency; - uint64_t rem = ticks % tick_frequency; - - return sec * 1000000ULL + (rem * 1000000ULL) / tick_frequency; -} - -uint64_t os_tick_ns(void) -{ - uint64_t ticks = os_tick(); - uint64_t sec = ticks / tick_frequency; - uint64_t rem = ticks % tick_frequency; - - return sec * 1000000000ULL + (rem * 1000000000ULL) / tick_frequency; -} - int os_usleep(uint64_t usec) { while (usec > 0) { @@ -4556,27 +4534,26 @@ void os_abort(void) size_t size; unsigned i; - fprintf(stderr, "Stacktrace of " PACKAGE " v" VERSION); - fprintf(stderr, ", mingw"); + printf("Stacktrace of " PACKAGE " v" VERSION); + printf(", mingw"); #ifdef __GNUC__ - fprintf(stderr, ", gcc " __VERSION__); + printf(", gcc " __VERSION__); #endif - fprintf(stderr, ", %d-bit", (int)sizeof(void*) * 8); - fprintf(stderr, ", PATH_MAX=%d", PATH_MAX); - fprintf(stderr, "\n"); + printf(", %d-bit", (int)sizeof(void*) * 8); + printf(", PATH_MAX=%d", PATH_MAX); + printf("\n"); /* get stackstrace, but without symbols */ - size = CaptureStackBackTrace(0, 32, stack, 0); + size = CaptureStackBackTrace(0, 32, stack, NULL); for (i = 0; i < size; ++i) - fprintf(stderr, "[bt] %02u: %p\n", i, stack[i]); + printf("[bt] %02u: %p\n", i, stack[i]); - fprintf(stderr, "Please report this error to the SnapRAID Issues:\n"); - fprintf(stderr, "https://github.com/amadvance/snapraid/issues\n"); - fflush(stderr); + printf("Please report this error to the SnapRAID Issues:\n"); + printf("https://github.com/amadvance/snapraid/issues\n"); - /* use exit(134) (128 + SIGABRT) and not abort to avoid the Windows abort dialog while distinguishing abort from normal failure */ - exit(134); + /* use exit() and not abort to avoid the Windows abort dialog */ + os_exit(); } void os_exit(void) @@ -4628,13 +4605,6 @@ void os_init(unsigned opt) os_exit(); } - LARGE_INTEGER freq; - if (!QueryPerformanceFrequency(&freq) || freq.QuadPart <= 0) { - os_syslog(OS_LVL_CRITICAL, "error calling QueryPerformanceFrequency()"); - os_exit(); - } - tick_frequency = freq.QuadPart; - if (windows_mutex_init(&exec_mutex, 0) != 0) { os_syslog(OS_LVL_CRITICAL, "error calling windows_mutex_init()"); os_exit(); @@ -4661,33 +4631,21 @@ void os_init(unsigned opt) os_exit(); } - /* - * Skip creating a separate job object if already running inside one - * (e.g. spawned by snapraid-daemon, a service, or Task Scheduler). - * Child processes automatically inherit the parent's job object, and - * attempting to assign them to a second independent job object would - * fail with ERROR_ACCESS_DENIED. - */ - BOOL in_job = 0; - if (IsProcessInJob(GetCurrentProcess(), 0, &in_job) && in_job) { - os_job_handle = 0; - } else { - /* initialize job object to terminate child processes if daemon dies */ - os_job_handle = CreateJobObjectW(0, 0); - if (os_job_handle != 0) { - JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; - ZeroMemory(&info, sizeof(info)); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if (!SetInformationJobObject(os_job_handle, JobObjectExtendedLimitInformation, &info, sizeof(info))) { - windows_errno(GetLastError()); - os_syslog(OS_LVL_WARNING, "failed to configure job object, errno=%s(%d)", strerror(errno), errno); - CloseHandle(os_job_handle); - os_job_handle = 0; - } - } else { + /* initialize job object to terminate child processes if daemon dies */ + os_job_handle = CreateJobObjectW(0, 0); + if (os_job_handle != 0) { + JOBOBJECT_EXTENDED_LIMIT_INFORMATION info; + ZeroMemory(&info, sizeof(info)); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if (!SetInformationJobObject(os_job_handle, JobObjectExtendedLimitInformation, &info, sizeof(info))) { windows_errno(GetLastError()); - os_syslog(OS_LVL_WARNING, "failed to create job object, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_WARNING, "failed to configure job object, errno=%s(%d)", strerror(errno), errno); + CloseHandle(os_job_handle); + os_job_handle = 0; } + } else { + windows_errno(GetLastError()); + os_syslog(OS_LVL_WARNING, "failed to create job object, errno=%s(%d)", strerror(errno), errno); } /* check for Wine presence */ @@ -4722,7 +4680,6 @@ void os_done(void) windows_mutex_destroy(&exec_mutex); windows_mutex_destroy(&tick_lock); - tick_frequency = 0; /* restore the normal execution level */ SetThreadExecutionState(WIN32_ES_CONTINUOUS); @@ -4731,4 +4688,3 @@ void os_done(void) } #endif - diff --git a/os/mingw.h b/os/mingw.h index 3dcbd7ef..13474359 100644 --- a/os/mingw.h +++ b/os/mingw.h @@ -4,7 +4,7 @@ #ifndef __PORTABLE_MINGW_H #define __PORTABLE_MINGW_H -#ifdef __MINGW32__ /* only for MingW */ +#ifdef __MINGW32__ /* Only for MingW */ #include @@ -98,7 +98,7 @@ wchar_t* u8tou16_mayfail(wchar_t* conv_buf, size_t number_of_wchar, const char* #undef PATH_MAX #define PATH_MAX 1024 -/* remap functions and types */ +/* Remap functions and types */ #undef fopen #define fopen windows_fopen #define FOPEN_TEXT "t" @@ -167,11 +167,11 @@ wchar_t* u8tou16_mayfail(wchar_t* conv_buf, size_t number_of_wchar, const char* #undef sleep #define sleep windows_sleep /* 4==DIR, 5,6,7=free, 8==REG */ -#define S_IFLNK 0x5000 /* symbolic link to file */ +#define S_IFLNK 0x5000 /* Symbolic link to file */ #define S_ISLNK(m) (((m) & _S_IFMT) == S_IFLNK) -#define S_IFLNKDIR 0x6000 /* symbolic link to directory */ +#define S_IFLNKDIR 0x6000 /* Symbolic link to directory */ #define S_ISLNKDIR(m) (((m) & _S_IFMT) == S_IFLNKDIR) -#define S_IFJUN 0x7000 /* junction */ +#define S_IFJUN 0x7000 /* Junction */ #define S_ISJUN(m) (((m) & _S_IFMT) == S_IFJUN) #undef readlink #define readlink windows_readlink @@ -210,7 +210,7 @@ wchar_t* u8tou16_mayfail(wchar_t* conv_buf, size_t number_of_wchar, const char* */ #define STAT_NSEC_INVALID -1 -/* we have nano second support */ +/* We have nano second support */ #define STAT_NSEC(st) ((int)(st)->st_mtimensec) static inline int WEXITSTATUS(DWORD status) diff --git a/os/os.h b/os/os.h index 8befff56..a0544181 100644 --- a/os/os.h +++ b/os/os.h @@ -397,18 +397,6 @@ uint64_t os_tick(void); */ uint64_t os_tick_ms(void); -/** - * Get the os_tick counter value in microsecond. - * \return Monotonic clock value in microseconds. - */ -uint64_t os_tick_us(void); - -/** - * Get the os_tick counter value in nanosecond. - * \return Monotonic clock value in nanoseconds. - */ -uint64_t os_tick_ns(void); - /** * Get the os_tick counter value in seconds. * \return Monotonic clock value in seconds. diff --git a/os/portable.h b/os/portable.h index 516c24cf..1dcd0963 100644 --- a/os/portable.h +++ b/os/portable.h @@ -5,11 +5,11 @@ #define __PORTABLE_H #if HAVE_CONFIG_H -#include "config.h" /* use " to include first in the same directory of this file */ +#include "config.h" /* Use " to include first in the same directory of this file */ #endif /***************************************************************************/ -/* config */ +/* Config */ #ifdef __MINGW32__ /** @@ -70,22 +70,6 @@ #define __noreturn __attribute__((noreturn)) #endif -#ifndef __aligned -#if defined(__GNUC__) || defined(__clang__) -#define __aligned(n) __attribute__((aligned(n))) -#else -#define __aligned(n) -#endif -#endif - -#ifndef __aligned_loops -#if defined(__GNUC__) && !defined(__clang__) -#define __aligned_loops(n) __attribute__((optimize("align-loops=" #n))) -#else -#define __aligned_loops(n) -#endif -#endif - /** * Architecture for inline assembly. */ @@ -100,22 +84,11 @@ #define CONFIG_X86_64 1 #endif -#if defined(__aarch64__) -#if defined(__ARM_NEON) || defined(__ARM_NEON__) -#define CONFIG_NEON 1 -#endif -#if defined(__ARM_FEATURE_CRC32) +#if defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) #define CONFIG_ARM_CRC 1 #endif #endif -#if defined(__arm__) -#if defined(__ARM_NEON) || defined(__ARM_NEON__) -#define CONFIG_NEON32 1 -#endif -#endif -#endif - /** * Includes some platform specific headers. */ @@ -159,11 +132,12 @@ #include #endif + /** * Includes some standard headers. */ #include -#include /* on many systems (e.g., Darwin), `stdio.h' is a prerequisite. */ +#include /* On many systems (e.g., Darwin), `stdio.h' is a prerequisite. */ #include #include #include @@ -200,6 +174,10 @@ #endif #endif +#if HAVE_MACH_MACH_TIME_H +#include +#endif + #if HAVE_DIRENT_H #include #endif @@ -299,7 +277,7 @@ #include #endif -/** +/* * Flags for open() */ #ifndef O_PATH diff --git a/os/unix.c b/os/unix.c index 6ba1eee6..bc8e9ae0 100644 --- a/os/unix.c +++ b/os/unix.c @@ -3,7 +3,7 @@ #include "os/portable.h" -#ifndef __MINGW32__ /* only for Unix */ +#ifndef __MINGW32__ /* Only for Unix */ #include "os.h" @@ -353,7 +353,7 @@ int filephy(const char* path, uint64_t size, uint64_t* physical) /****************************************************************************/ /* exec */ -/** +/* * Scrubbed environment * Only provide the bare essentials. */ @@ -369,7 +369,7 @@ static char* const envp_scrubbed[] = { NULL }; -/** +/* * Enforce a strict 128-byte shebang limit to match the actual Linux kernel behavior * and prevent security risks from truncation. * @@ -461,24 +461,24 @@ static int verify_shebang_interpreter(int fd, const char* script_path) bytes_read = pread(fd, shebang, sizeof(shebang) - 1, 0); /* reserve space for the terminating 0 */ if (bytes_read < 0) { - os_syslog(OS_LVL_ERROR, "failed to read script shebang, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to read script shebang, path=%s, errno=%s(%d)", script_path, strerror(errno), errno); return -1; } if (bytes_read < 4) { - os_syslog(OS_LVL_ERROR, "script %s is too small or missing a shebang", script_path); + os_syslog(OS_LVL_INFO, "script %s is too small or missing a shebang", script_path); return -1; } shebang[bytes_read] = 0; /* check for shebang */ if (shebang[0] != '#' || shebang[1] != '!') { - os_syslog(OS_LVL_ERROR, "script %s is missing shebang (#!)", script_path); + os_syslog(OS_LVL_INFO, "script %s is missing shebang (#!)", script_path); return -1; } char* end_of_line = strchr(shebang, '\n'); if (!end_of_line || end_of_line - shebang > 128) { - os_syslog(OS_LVL_ERROR, "script %s has invalid or overlong shebang (#!), exceeds 126 characters", script_path); + os_syslog(OS_LVL_INFO, "script %s has invalid or overlong shebang (#!), exceeds 126 characters", script_path); return -1; } *end_of_line = 0; @@ -489,7 +489,7 @@ static int verify_shebang_interpreter(int fd, const char* script_path) ++interpreter; if (*interpreter == 0) { - os_syslog(OS_LVL_ERROR, "script %s has empty shebang", script_path); + os_syslog(OS_LVL_INFO, "script %s has empty shebang", script_path); return -1; } @@ -510,49 +510,49 @@ static int verify_shebang_interpreter(int fd, const char* script_path) } if (!found) { - os_syslog(OS_LVL_ERROR, "script %s uses disallowed interpreter %s", script_path, interpreter); + os_syslog(OS_LVL_INFO, "script %s uses disallowed interpreter %s", script_path, interpreter); return -1; } /* verify interpreter exists and is safe */ if (stat(interpreter, &st) != 0) { - os_syslog(OS_LVL_ERROR, "interpreter %s does not exist, errno=%s(%d)", interpreter, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "interpreter %s does not exist, errno=%s(%d)", interpreter, strerror(errno), errno); return -1; } /* interpreter must be a regular file */ if (!S_ISREG(st.st_mode)) { - os_syslog(OS_LVL_ERROR, "interpreter %s must be a regular file", interpreter); + os_syslog(OS_LVL_INFO, "interpreter %s must be a regular file", interpreter); return -1; } /* interpreter must be owned by root */ if (st.st_uid != 0) { - os_syslog(OS_LVL_ERROR, "interpreter %s not owned by root", interpreter); + os_syslog(OS_LVL_INFO, "interpreter %s not owned by root", interpreter); return -1; } /* interpreter must be not world-writable */ if (st.st_mode & S_IWOTH) { - os_syslog(OS_LVL_ERROR, "interpreter %s is world-writable", interpreter); + os_syslog(OS_LVL_INFO, "interpreter %s is world-writable", interpreter); return -1; } /* interpreter must be not group-writable (unless group is root) */ if ((st.st_mode & S_IWGRP) && st.st_gid != 0) { - os_syslog(OS_LVL_ERROR, "interpreter %s is group-writable by non-root group", interpreter); + os_syslog(OS_LVL_INFO, "interpreter %s is group-writable by non-root group", interpreter); return -1; } /* interpreter must be executable */ if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { - os_syslog(OS_LVL_ERROR, "interpreter %s is not executable", interpreter); + os_syslog(OS_LVL_INFO, "interpreter %s is not executable", interpreter); return -1; } /* interpreter must be not setuid / setgid */ if (st.st_mode & (S_ISUID | S_ISGID)) { - os_syslog(OS_LVL_ERROR, "file %s has setuid/setgid bits set", interpreter); + os_syslog(OS_LVL_INFO, "file %s has setuid/setgid bits set", interpreter); return -1; } @@ -560,7 +560,7 @@ static int verify_shebang_interpreter(int fd, const char* script_path) return 0; } -/** +/* * Securely verify and open an executable file. * * Performs a series of security checks on the file at @exec_path before @@ -595,7 +595,7 @@ static int verify_shebang_interpreter(int fd, const char* script_path) * * Returns an open file descriptor (>= 0) on success. The caller is * responsible for closing it. Returns -1 on any verification failure; - * the specific reason is emitted via os_syslog(OS_LVL_ERROR, ...). + * the specific reason is emitted via os_syslog(OS_LVL_INFO, ...). */ static int verify_executable(const char* exec_path, char* resolved_path, int is_script) { @@ -610,29 +610,29 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ /* verify path is absolute */ if (exec_path[0] != '/') { - os_syslog(OS_LVL_ERROR, "path %s must be absolute", exec_path); + os_syslog(OS_LVL_INFO, "path %s must be absolute", exec_path); return -1; } /* resolve the path to prevent symlink attacks */ if (!realpath(exec_path, resolved_path)) { - os_syslog(OS_LVL_ERROR, "failed to resolve %s, errno=%s(%d)", exec_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to resolve %s, errno=%s(%d)", exec_path, strerror(errno), errno); return -1; } char* last_slash = strrchr(resolved_path, '/'); if (last_slash == 0) { - os_syslog(OS_LVL_ERROR, "relative execution of %s not allowed", resolved_path); + os_syslog(OS_LVL_INFO, "relative execution of %s not allowed", resolved_path); return -1; } if (last_slash == resolved_path) { - os_syslog(OS_LVL_ERROR, "root dir execution of %s not allowed", resolved_path); + os_syslog(OS_LVL_INFO, "root dir execution of %s not allowed", resolved_path); return -1; } const char* exec_name = last_slash + 1; if (exec_name[0] == 0) { - os_syslog(OS_LVL_ERROR, "no executable name in %s", exec_path); + os_syslog(OS_LVL_INFO, "no executable name in %s", exec_path); return -1; } @@ -643,33 +643,33 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ int dir_fd = open(dir_path, O_PATH | O_DIRECTORY | O_NOFOLLOW | O_CLOEXEC); if (dir_fd < 0) { - os_syslog(OS_LVL_ERROR, "failed to open directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); return -1; } if (fstat(dir_fd, &st) != 0) { - os_syslog(OS_LVL_ERROR, "failed to stat directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to stat directory %s, errno=%s(%d)", dir_path, strerror(errno), errno); close(dir_fd); return -1; } /* directory must be owned by root or the daemon's real user */ if (st.st_uid != process_uid && st.st_uid != process_euid && st.st_uid != 0) { - os_syslog(OS_LVL_ERROR, "directory %s owner must match the daemon owner or be root", dir_path); + os_syslog(OS_LVL_INFO, "directory %s owner must match the daemon owner or be root", dir_path); close(dir_fd); return -1; } /* directory must be not group-writable unless group matches daemon */ if ((st.st_mode & S_IWGRP) && st.st_gid != process_gid && st.st_gid != process_egid && st.st_gid != 0) { - os_syslog(OS_LVL_ERROR, "directory %s must be not group-writable unless group matches daemon owner or root", dir_path); + os_syslog(OS_LVL_INFO, "directory %s must be not group-writable unless group matches daemon owner or root", dir_path); close(dir_fd); return -1; } /* directory must be not world-writable */ if (st.st_mode & S_IWOTH) { - os_syslog(OS_LVL_ERROR, "directory %s must be not world-writable", dir_path); + os_syslog(OS_LVL_INFO, "directory %s must be not world-writable", dir_path); close(dir_fd); return -1; } @@ -689,7 +689,7 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ int fd = openat(dir_fd, exec_name, flags); if (fd < 0) { - os_syslog(OS_LVL_ERROR, "failed to open %s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open %s, errno=%s(%d)", resolved_path, strerror(errno), errno); close(dir_fd); return -1; } @@ -698,14 +698,14 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ /* get the file handle (TOCTOU Protection) */ if (fstat(fd, &st) == -1) { - os_syslog(OS_LVL_ERROR, "failed to stat %s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to stat %s, errno=%s(%d)", resolved_path, strerror(errno), errno); close(fd); return -1; } /* ensure it's a regular file */ if (!S_ISREG(st.st_mode)) { - os_syslog(OS_LVL_ERROR, "file %s is not a regular file", resolved_path); + os_syslog(OS_LVL_INFO, "file %s is not a regular file", resolved_path); close(fd); return -1; } @@ -714,7 +714,7 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ if (!is_script) { char magic[2]; if (read(fd, magic, 2) == 2 && magic[0] == '#' && magic[1] == '!') { - os_syslog(OS_LVL_ERROR, "file %s is a script, which is not supported", resolved_path); + os_syslog(OS_LVL_INFO, "file %s is a script, which is not supported", resolved_path); close(fd); return -1; } @@ -725,35 +725,35 @@ static int verify_executable(const char* exec_path, char* resolved_path, int is_ /* ensure it has execute permissions */ if (!(st.st_mode & (S_IXUSR | S_IXGRP | S_IXOTH))) { - os_syslog(OS_LVL_ERROR, "file %s is not an executable", resolved_path); + os_syslog(OS_LVL_INFO, "file %s is not an executable", resolved_path); close(fd); return -1; } /* must be owned by root or the daemon's real user */ if (st.st_uid != process_uid && st.st_uid != process_euid && st.st_uid != 0) { - os_syslog(OS_LVL_ERROR, "file %s owner must match the daemon owner or be root", resolved_path); + os_syslog(OS_LVL_INFO, "file %s owner must match the daemon owner or be root", resolved_path); close(fd); return -1; } /* must be not group-writable unless group matches daemon */ if ((st.st_mode & S_IWGRP) && st.st_gid != process_gid && st.st_gid != process_egid && st.st_gid != 0) { - os_syslog(OS_LVL_ERROR, "file %s must be not group-writable unless group matches daemon owner or root", resolved_path); + os_syslog(OS_LVL_INFO, "file %s must be not group-writable unless group matches daemon owner or root", resolved_path); close(fd); return -1; } /* must be not world-writable */ if (st.st_mode & S_IWOTH) { - os_syslog(OS_LVL_ERROR, "file %s must be not world-writable", resolved_path); + os_syslog(OS_LVL_INFO, "file %s must be not world-writable", resolved_path); close(fd); return -1; } /* must be not setuid / setgid */ if (st.st_mode & (S_ISUID | S_ISGID)) { - os_syslog(OS_LVL_ERROR, "file %s has setuid/setgid bits set", resolved_path); + os_syslog(OS_LVL_INFO, "file %s has setuid/setgid bits set", resolved_path); close(fd); return -1; } @@ -765,13 +765,13 @@ int os_validate_exec_input(const char* str) { /* reject paths trying to go up levels */ if (strstr(str, "..") != 0) { - os_syslog(OS_LVL_ERROR, "invalid argument %s for execution", str); + os_syslog(OS_LVL_INFO, "invalid argument %s for execution", str); return -1; } /* reject inputs starting with '-' to prevent Flag Injection */ if (str[0] == '-') { - os_syslog(OS_LVL_ERROR, "invalid argument %s for execution", str); + os_syslog(OS_LVL_INFO, "invalid argument %s for execution", str); return -1; } @@ -810,7 +810,7 @@ static int os_user_lookup(const char* run_as_user, struct os_user* user) char pw_buf[PASSWD_MAX]; int err = getpwnam_r(run_as_user, &pw, pw_buf, sizeof(pw_buf), &pw_res); if (err != 0 || !pw_res) { - os_syslog(OS_LVL_ERROR, "failed to get user info for %s, errno=%s(%d)", run_as_user, strerror(err ? err : ENOENT), err ? err : ENOENT); + os_syslog(OS_LVL_INFO, "failed to get user info for %s, errno=%s(%d)", run_as_user, strerror(err ? err : ENOENT), err ? err : ENOENT); return -1; } @@ -822,7 +822,7 @@ static int os_user_lookup(const char* run_as_user, struct os_user* user) int ngroups = sizeof(groups_buf) / sizeof(groups_buf[0]); if (getgrouplist(pw.pw_name, (os_group_t)pw.pw_gid, groups_buf, &ngroups) < 0) { - os_syslog(OS_LVL_ERROR, "failed to get group list for %s, exceeds maximum of %d groups", run_as_user, GROUPS_MAX); + os_syslog(OS_LVL_INFO, "failed to get group list for %s, exceeds maximum of %d groups", run_as_user, GROUPS_MAX); return -1; } @@ -940,7 +940,7 @@ static int pid_kill(pid_t pid) return -1; } - /* send SIGKILL signal to the negative PID to target the entire Process Group */ + /* Send SIGKILL signal to the negative PID to target the entire Process Group */ int ret = kill(-pid, SIGKILL); /* @@ -1198,7 +1198,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou pid = fork(); if (pid < 0) { - os_syslog(OS_LVL_ERROR, "failed to fork script=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to fork script=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); pid_unpublish(pid_slot, 0); close(fd); return -1; @@ -1219,7 +1219,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou os_user_drop_privileges(&user); #if defined(__linux__) && defined(PR_SET_PDEATHSIG) - /* send SIGKILL to child if parent daemon dies unexpectedly */ + /* Send SIGKILL to child if parent daemon dies unexpectedly */ prctl(PR_SET_PDEATHSIG, SIGKILL); /* exit if parent died before prctl to avoid becoming orphaned */ if (getppid() != ppid) { @@ -1323,7 +1323,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou } if (ret == -1) { - os_syslog(OS_LVL_ERROR, "failed to wait for script, path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to wait for script, path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); return -1; } @@ -1331,7 +1331,7 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_ERROR, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); + os_syslog(OS_LVL_INFO, "script %s timeout after %" PRId64 " seconds", resolved_path, execution_time); return 128 + SIGALRM; } @@ -1344,17 +1344,17 @@ int os_script(char** argv, char** envp, const char* run_as_user, uint64_t timeou if (exit_code == 0) os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with success", resolved_path, execution_time); else - os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); + os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with exit code %d", resolved_path, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds with signal %s(%d)", resolved_path, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Linux it should never happen */ - os_syslog(OS_LVL_ERROR, "script %s terminated in %" PRId64 " seconds for unknown reason, status=%d", resolved_path, execution_time, status); + os_syslog(OS_LVL_INFO, "script %s terminated in %" PRId64 " seconds for unknown reason, status=%d", resolved_path, execution_time, status); return -1; } } @@ -1373,14 +1373,14 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t /* create pipe only if we have text to send */ if (stdin_text != 0) { if (pipe_cloexec(pipe_fds) < 0) { - os_syslog(OS_LVL_ERROR, "failed to create input pipe, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to create input pipe, errno=%s(%d)", strerror(errno), errno); return -1; } int flags = fcntl(pipe_fds[1], F_GETFL); if (flags < 0 || fcntl(pipe_fds[1], F_SETFL, flags | O_NONBLOCK) < 0) { - os_syslog(OS_LVL_ERROR, "failed to make input pipe non-blocking, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to make input pipe non-blocking, errno=%s(%d)", strerror(errno), errno); close(pipe_fds[0]); close(pipe_fds[1]); return -1; @@ -1411,7 +1411,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t pid = fork(); if (pid < 0) { - os_syslog(OS_LVL_ERROR, "failed to fork command=%s, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to fork command=%s, errno=%s(%d)", command, strerror(errno), errno); pid_unpublish(pid_slot, 0); if (pipe_fds[0] != -1) { close(pipe_fds[0]); @@ -1432,13 +1432,13 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t setpgid(0, 0); if (pipe_fds[1] != -1) - close(pipe_fds[1]); /* close unused write end */ + close(pipe_fds[1]); /* Close unused write end */ /* drop privileges first (if configured) */ os_user_drop_privileges(&user); #if defined(__linux__) && defined(PR_SET_PDEATHSIG) - /* send SIGKILL to child if parent daemon dies unexpectedly */ + /* Send SIGKILL to child if parent daemon dies unexpectedly */ prctl(PR_SET_PDEATHSIG, SIGKILL); /* exit if parent died before prctl to avoid becoming orphaned */ if (getppid() != ppid) { @@ -1461,7 +1461,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t _exit(126); } - /* redirect STDOUT and STDERR to /dev/null */ + /* Redirect STDOUT and STDERR to /dev/null */ if (dup2(null_fd, STDOUT_FILENO) == -1 || dup2(null_fd, STDERR_FILENO) == -1) _exit(126); @@ -1509,7 +1509,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t pid_unpublish(pid_slot, pid); if (input_error != 0) - os_syslog(OS_LVL_ERROR, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(input_error), input_error); + os_syslog(OS_LVL_INFO, "failed to write stdin to command %s, errno=%s(%d)", command, strerror(input_error), input_error); if (ret != -1) { do { @@ -1518,7 +1518,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t } if (ret == -1) { - os_syslog(OS_LVL_ERROR, "failed to wait for command, command=%s, errno=%s(%d)", command, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to wait for command, command=%s, errno=%s(%d)", command, strerror(errno), errno); return -1; } @@ -1526,7 +1526,7 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t int64_t execution_time = stop - start; if (timed_out) { - os_syslog(OS_LVL_ERROR, "command %s timeout after %" PRId64 " seconds", command, execution_time); + os_syslog(OS_LVL_INFO, "command %s timeout after %" PRId64 " seconds", command, execution_time); return 128 + SIGALRM; } @@ -1538,16 +1538,16 @@ int os_command(const char* command, const char* run_as_user, const char* stdin_t if (exit_code == 0) os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with success", command, execution_time); else - os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); + os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with exit code %d", command, execution_time, exit_code); return exit_code; } else if (WIFSIGNALED(status)) { /* child died from a signal */ int sig = WTERMSIG(status); - os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); + os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds with signal %s(%d)", command, execution_time, os_signal_name(sig), sig); return 128 + sig; } else { /* in Linux it should never happen */ - os_syslog(OS_LVL_ERROR, "command %s terminated in %" PRId64 " seconds for unknown reason, status=%d", command, execution_time, status); + os_syslog(OS_LVL_INFO, "command %s terminated in %" PRId64 " seconds for unknown reason, status=%d", command, execution_time, status); return -1; } } @@ -1584,7 +1584,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char if (has_out) { if (pipe_cloexec(out_pipe) < 0) { - os_syslog(OS_LVL_ERROR, "failed to open output pipe, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open output pipe, errno=%s(%d)", strerror(errno), errno); close(fd); return -1; } @@ -1592,7 +1592,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char if (has_err) { if (pipe_cloexec(err_pipe) < 0) { - os_syslog(OS_LVL_ERROR, "failed to open error pipe, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to open error pipe, errno=%s(%d)", strerror(errno), errno); if (has_out) { close(out_pipe[0]); close(out_pipe[1]); @@ -1634,7 +1634,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char pid = fork(); if (pid < 0) { - os_syslog(OS_LVL_ERROR, "failed to fork path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to fork path=%s, errno=%s(%d)", resolved_path, strerror(errno), errno); pid_unpublish(pid_slot, 0); if (has_out) { close(out_pipe[0]); @@ -1657,7 +1657,7 @@ pid_t os_spawn(char** argv, int* stdout_read_fd, int* stderr_read_fd, const char os_user_drop_privileges(&user); #if defined(__linux__) && defined(PR_SET_PDEATHSIG) - /* send SIGKILL to child if parent daemon dies unexpectedly */ + /* Send SIGKILL to child if parent daemon dies unexpectedly */ prctl(PR_SET_PDEATHSIG, SIGKILL); /* exit if parent died before prctl to avoid becoming orphaned */ if (getppid() != ppid) { @@ -1819,7 +1819,7 @@ pid_t os_wait(pid_t pid, int* status, pid_t* pid_slot) } while (ret == -1 && errno == EINTR); if (ret == -1) { - os_syslog(OS_LVL_ERROR, "failed to wait, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to wait, errno=%s(%d)", strerror(errno), errno); } return ret; @@ -1996,7 +1996,7 @@ int os_privileges_drop(void) /* find the unprivileged user "nobody" */ struct passwd* pw = getpwnam("nobody"); if (!pw) { - os_syslog(OS_LVL_ERROR, "unprivileged user 'nobody' not found"); + os_syslog(OS_LVL_INFO, "unprivileged user 'nobody' not found"); return -1; } @@ -2010,7 +2010,7 @@ int os_privileges_drop(void) */ privileged_group_count = getgroups(0, 0); if (privileged_group_count < 0) { - os_syslog(OS_LVL_ERROR, "failed to get supplementary group count, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to get supplementary group count, errno=%s(%d)", strerror(errno), errno); return -1; } @@ -2022,12 +2022,12 @@ int os_privileges_drop(void) } if (getgroups(privileged_group_count, privileged_groups) != privileged_group_count) { - os_syslog(OS_LVL_ERROR, "failed to get supplementary groups, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to get supplementary groups, errno=%s(%d)", strerror(errno), errno); return -1; } if (setgroups(0, 0) != 0) { - os_syslog(OS_LVL_ERROR, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); return -1; } @@ -2035,14 +2035,14 @@ int os_privileges_drop(void) } if (setegid(unpriv_gid) != 0) { - os_syslog(OS_LVL_ERROR, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); return -1; } gid_dropped = 1; if (seteuid(unpriv_uid) != 0) { - os_syslog(OS_LVL_ERROR, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); return -1; } @@ -2072,7 +2072,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2083,7 +2083,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2094,7 +2094,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2105,7 +2105,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to acquire privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2116,7 +2116,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to acquire group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2127,7 +2127,7 @@ void os_privileges_acquire(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to acquire supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to acquire supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2147,7 +2147,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2158,19 +2158,19 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } } - /* drop UID last. */ + /* Drop UID last. */ if (uid_dropped) { if (syscall(SYS_setresuid, (uid_t)-1, (uid_t)unpriv_uid, (uid_t)-1) != 0) { if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2181,7 +2181,7 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release supplementary group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release supplementary group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2192,19 +2192,19 @@ void os_privileges_release(void) if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release group privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release group privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } } - /* drop UID last. */ + /* Drop UID last. */ if (uid_dropped) { if (seteuid(unpriv_uid) != 0) { if (errno == EPERM) { os_syslog(OS_LVL_INFO, "permission denied to release privileges, continuing with active privileges"); } else { - os_syslog(OS_LVL_ERROR, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); + os_syslog(OS_LVL_INFO, "failed to release privileges, errno=%s(%d)", strerror(errno), errno); os_abort(); } } @@ -2217,18 +2217,17 @@ void os_privileges_release(void) uint64_t os_tick(void) { -#if HAVE_CLOCK_GETTIME && (defined(CLOCK_UPTIME_RAW) || defined(CLOCK_MONOTONIC_RAW) || defined(CLOCK_MONOTONIC)) +#if HAVE_MACH_ABSOLUTE_TIME + /* for Mac OS X */ + return mach_absolute_time(); +#elif HAVE_CLOCK_GETTIME && (defined(CLOCK_MONOTONIC) || defined(CLOCK_MONOTONIC_RAW)) + /* for Linux */ struct timespec tv; /* nanosecond precision with clock_gettime() */ -#if defined(CLOCK_UPTIME_RAW) - /* for macOS (without sleep and without NTP slewing) */ - if (clock_gettime(CLOCK_UPTIME_RAW, &tv) != 0) { -#elif defined(CLOCK_MONOTONIC_RAW) - /* for Linux (without sleep and without NTP slewing) */ +#if defined(CLOCK_MONOTONIC_RAW) if (clock_gettime(CLOCK_MONOTONIC_RAW, &tv) != 0) { #else - /* for other POSIX */ if (clock_gettime(CLOCK_MONOTONIC, &tv) != 0) { #endif /* LCOV_EXCL_START */ @@ -2248,7 +2247,7 @@ uint64_t os_tick(void) /* LCOV_EXCL_STOP */ } - return tv.tv_sec * 1000000000ULL + tv.tv_usec * 1000ULL; + return tv.tv_sec * 1000000ULL + tv.tv_usec; #endif } @@ -2277,16 +2276,6 @@ uint64_t os_tick_ms(void) #endif } -uint64_t os_tick_us(void) -{ - return os_tick() / 1000; -} - -uint64_t os_tick_ns(void) -{ - return os_tick(); -} - uint64_t os_tick_sec(void) { #if HAVE_CLOCK_GETTIME && defined(CLOCK_MONOTONIC) @@ -2338,6 +2327,8 @@ int os_usleep(uint64_t usec) return 0; } + + int os_randomize(void* ptr, size_t size) { int f; diff --git a/os/unix.h b/os/unix.h index 5ac326aa..91775d54 100644 --- a/os/unix.h +++ b/os/unix.h @@ -67,7 +67,7 @@ static inline int close_range_impl(unsigned int first, unsigned int last, unsign */ #define STAT_NSEC_INVALID -1 -/* check if we have nanoseconds support */ +/* Check if we have nanoseconds support */ #if HAVE_STRUCT_STAT_ST_MTIM_TV_NSEC #define STAT_NSEC(st) ((int)(st)->st_mtim.tv_nsec) /* Linux */ #elif HAVE_STRUCT_STAT_ST_MTIMENSEC diff --git a/raid/NOTES b/raid/NOTES index b095fc2a..20028207 100644 --- a/raid/NOTES +++ b/raid/NOTES @@ -1055,12 +1055,6 @@ The implementation deliberately targets at most six parity levels. Short list of ideas already tried or rejected --------------------------------------------- -* Loop alignment to 64 bytes: - improves scalar integer routines (raid/int.c) by +4% to +15% by avoiding fetch - and decode stalls on compact loops. Tried on vector SIMD routines (SSE, AVX, - NEON), with no improvement because wide unrolled loops are memory-bound and - cached in the uop cache. Kept only for integer routines. - * AVX512 128-byte generation processing: tried, can be slower. diff --git a/raid/combo.h b/raid/combo.h index 7c833430..ff59be03 100644 --- a/raid/combo.h +++ b/raid/combo.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_COMBO_H -#define RAID_COMBO_H +#ifndef __RAID_COMBO_H +#define __RAID_COMBO_H #include diff --git a/raid/cpu.h b/raid/cpu.h index 418c1c05..0933133a 100644 --- a/raid/cpu.h +++ b/raid/cpu.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_CPU_H -#define RAID_CPU_H +#ifndef __RAID_CPU_H +#define __RAID_CPU_H #ifdef CONFIG_X86 diff --git a/raid/gf.h b/raid/gf.h index dce476cb..0e92d508 100644 --- a/raid/gf.h +++ b/raid/gf.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_GF_H -#define RAID_GF_H +#ifndef __RAID_GF_H +#define __RAID_GF_H /* * Galois field operations. diff --git a/raid/helper.h b/raid/helper.h index 15cbaa46..f9ca609e 100644 --- a/raid/helper.h +++ b/raid/helper.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_HELPER_H -#define RAID_HELPER_H +#ifndef __RAID_HELPER_H +#define __RAID_HELPER_H /** * Inserts an integer in a sorted vector. diff --git a/raid/int.c b/raid/int.c index 9587c4d4..0aa5148e 100644 --- a/raid/int.c +++ b/raid/int.c @@ -9,7 +9,6 @@ * * Uses 16-byte chunks across four 32-bit words. */ -__aligned_loops(64) void raid_gen1_int32(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -50,7 +49,6 @@ void raid_gen1_int32(int nd, size_t size, void **vv, int streaming) * * Uses 32-byte chunks across four 64-bit words. */ -__aligned_loops(64) void raid_gen1_int64(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -195,7 +193,6 @@ static __always_inline void raid_gen2_int64_gen(int nd, size_t size, void **vv, * * Uses Horner's method with 8-byte chunks across two 32-bit words. */ -__aligned_loops(64) void raid_genz_int32_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -253,7 +250,6 @@ void raid_genz_int32_raid(int nd, size_t size, void **vv, int streaming) * * Uses Horner's method with 16-byte chunks across two 64-bit words. */ -__aligned_loops(64) void raid_genz_int64_raid(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -311,7 +307,6 @@ void raid_genz_int64_raid(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -__aligned_loops(64) void raid_gen2_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -366,7 +361,6 @@ void raid_gen2_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -__aligned_loops(64) void raid_gen3_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -431,7 +425,6 @@ void raid_gen3_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -__aligned_loops(64) void raid_gen4_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -505,7 +498,6 @@ void raid_gen4_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -__aligned_loops(64) void raid_gen5_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -588,7 +580,6 @@ void raid_gen5_int8(int nd, size_t size, void **vv, int streaming) * * Precomputes per-disk multiplication tables and processes 2-byte chunks. */ -__aligned_loops(64) void raid_gen6_int8(int nd, size_t size, void **vv, int streaming) { uint8_t **v = (uint8_t **)vv; @@ -1017,35 +1008,30 @@ static __always_inline void raid_recX_int8(int nr, int has_p, int *id, int *ip, } } -__aligned_loops(64) void raid_gen2_int32_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int32_gen(nd, size, vv, 2); } -__aligned_loops(64) void raid_gen2_int32_aes(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int32_gen(nd, size, vv, 3); } -__aligned_loops(64) void raid_gen2_int64_raid(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int64_gen(nd, size, vv, 2); } -__aligned_loops(64) void raid_gen2_int64_aes(int nd, size_t size, void **vv, int streaming) { (void)streaming; raid_gen2_int64_gen(nd, size, vv, 3); } -__aligned_loops(64) void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); @@ -1056,7 +1042,6 @@ void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(1, 0, id, ip, nd, size, vv); } -__aligned_loops(64) void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); @@ -1067,7 +1052,6 @@ void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(2, 0, id, ip, nd, size, vv); } -__aligned_loops(64) void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 3); @@ -1077,7 +1061,6 @@ void raid_rec3_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(3, 0, id, ip, nd, size, vv); } -__aligned_loops(64) void raid_rec4_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 4); @@ -1087,7 +1070,6 @@ void raid_rec4_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(4, 0, id, ip, nd, size, vv); } -__aligned_loops(64) void raid_rec5_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 5); @@ -1097,7 +1079,6 @@ void raid_rec5_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_int8(5, 0, id, ip, nd, size, vv); } -__aligned_loops(64) void raid_rec6_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 6); diff --git a/raid/internal.h b/raid/internal.h index 3650665f..4fbe9b89 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_INTERNAL_H -#define RAID_INTERNAL_H +#ifndef __RAID_INTERNAL_H +#define __RAID_INTERNAL_H #if HAVE_CONFIG_H /* Includes the project configuration for HAVE_* defines */ @@ -61,22 +61,7 @@ * Forced alignment. */ #ifndef __aligned -#if defined(__GNUC__) || defined(__clang__) #define __aligned(a) __attribute__((aligned(a))) -#else -#define __aligned(a) -#endif -#endif - -/* - * Forced loops alignment. - */ -#ifndef __aligned_loops -#if defined(__GNUC__) && !defined(__clang__) -#define __aligned_loops(n) __attribute__((optimize("align-loops=" #n))) -#else -#define __aligned_loops(n) -#endif #endif /* diff --git a/raid/memory.h b/raid/memory.h index 7d73e0b3..19aa8af1 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_MEMORY_H -#define RAID_MEMORY_H +#ifndef __RAID_MEMORY_H +#define __RAID_MEMORY_H /** * Memory allocation and benchmark layout helpers. diff --git a/raid/raid.h b/raid/raid.h index 544d7cda..d4595a87 100644 --- a/raid/raid.h +++ b/raid/raid.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_H -#define RAID_H +#ifndef __RAID_H +#define __RAID_H #include diff --git a/raid/test.h b/raid/test.h index 1c0c99b4..bda0b456 100644 --- a/raid/test.h +++ b/raid/test.h @@ -1,8 +1,8 @@ // SPDX-License-Identifier: GPL-2.0-or-later // Copyright (C) 2013 Andrea Mazzoleni -#ifndef RAID_TEST_H -#define RAID_TEST_H +#ifndef __RAID_TEST_H +#define __RAID_TEST_H /** * Tests insertion function. diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 069de5c0..0d40894e 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -36,19 +36,19 @@ static int64_t diffgettimeofday(struct timeval *start, struct timeval *stop) d = 1000000LL * (stop->tv_sec - start->tv_sec); d += stop->tv_usec - start->tv_usec; - return d > 0 ? d : 0; + return d; } #define SPEED_START \ { \ - int speed_delta_ = delta; \ - int64_t elapsed_ = 0; \ - int64_t best_dt_ = INT64_MAX; \ - do { \ - struct timeval start; \ - struct timeval stop; \ - gettimeofday(&start, 0); \ - for (int i_ = 0; i_ < speed_delta_; ++i_) + int speed_delta_ = delta; \ + int64_t elapsed_ = 0; \ + int64_t best_dt_ = INT64_MAX; \ + do { \ + struct timeval start; \ + struct timeval stop; \ + gettimeofday(&start, 0); \ + for (int i_ = 0; i_ < speed_delta_; ++i_) #define SPEED_STOP \ gettimeofday(&stop, 0); \ diff --git a/smhasher3/README b/smhasher3/README index 74f7b5e9..91cb73c7 100644 --- a/smhasher3/README +++ b/smhasher3/README @@ -29,51 +29,13 @@ museair_snapraid.cpp MuseAir-128.bfast-snapraid.txt SMHasher3 output for museair_snapraid.cpp. - Result comparison: + Result comparison: The SnapRAID variant passes all 252 tests. The official MuseAir BFast-128 also passes its complete test set. The different short-input path used by SnapRAID shows no statistical regression. Small differences in the p-value distribution are not significant enough to rank either version as better. - Known collision analysis: - MuseAir v2 has two known adversarial collision constructions: - - 1. Short path: A published 32-byte message pair collides with rate - ~2^-17.5 over random seeds (and a similar rate for 128-bit functions - with independent 64-bit seed halves). - - 2. Long path without compression: An algebraic product-exchange - construction produces collisions for 80, 88, and 96-byte inputs when - no 96-byte compression round is executed. For MuseAir-128, the - published 80-byte pair collides only when seed_b == 0 (a 2^-64 weak-key - condition over uniformly random seeds). - - The product-exchange construction relies on relations between the initial - state words and the finalizer products. A single 96-byte compression round - destroys these relations; the construction does not survive when preceded - by a common 96-byte block or placed within a 192-byte message. - - Relevance to SnapRAID: - The known constructions do not currently affect SnapRAID's intended use: - - - The Short path is never used: SnapRAID routes all inputs through the - Long path. - - The no-compression construction does not survive SnapRAID's block format. - SnapRAID hashes full canonical RAID blocks (with zero-padding for partial - blocks). A typical block (e.g., 256 KiB) executes thousands of 96-byte - compression rounds, which destroy the product-exchange relations. - - While SnapRAID's fixed zero seed happens to satisfy the weak-key condition - seed_b == 0, finding collisions with a known seed in non-cryptographic - hashes is expected and does not indicate a weakness for detecting random - data corruption. - - References: - https://thomasahle.com/blog/adversarial-examples-for-hashes/ - https://thomasahle.com/blog/adversarial-examples-for-hashes/records/witness-searches/reanalysis-2026-09/README.html - https://thomasahle.com/blog/adversarial-examples-for-hashes/records/witness-searches/reanalysis-2026-09/museair-v2/README.html - spooky2_snapraid.cpp SpookyHash V2 128-bit as used by SnapRAID. SnapRAID always uses the Long path instead of the standard Short path below 192 bytes. @@ -81,48 +43,12 @@ spooky2_snapraid.cpp SpookyHash2-128-snapraid.txt SMHasher3 output for spooky2_snapraid.cpp. - Result comparison: + Result comparison: The SnapRAID variant passes all 252 tests. The official SpookyHash V2 128-bit implementation has strong BIC failures on several short input lengths. Those failures come from the Short path. Always using the Long path, as SnapRAID does, removes the observed BIC weakness. - Known collision analysis: - An adversarial collision has been demonstrated in the SpookyHash V2 Long - path itself. Published 275- and 286-byte message pairs introduce controlled - differences in the final 96-byte Mix block and cancel them in the End block. - - The required sign of the correction depends on a single internal state bit. - Two complementary pairs cover both possible values of this bit, each - colliding for approximately half (~2^-1) of randomly tested seeds. When the - cancellation succeeds, the entire 768-bit internal state matches before the - finalization rounds, yielding an identical 128-bit hash. - - Because the differential is injected in the final Mix block, the attack can - be extended to larger inputs by prepending common 96-byte blocks, provided - the final block preserves the required offset. - - Relevance to SnapRAID: - SnapRAID stores its 128-bit SpookyHash seed in the content file. Because the - seed is public, an attacker can determine the appropriate variant rather - than relying on a random ~2^-1 success rate. - - Furthermore, unlike MuseAir, SpookyHash does not pad partial blocks. A final - file block can therefore have arbitrary lengths (such as 275 or 286 bytes), - making the published lengths directly applicable. For full blocks, extending - the attack requires common 96-byte prefixes that preserve the required End - block structure. - - However, finding collisions with a public seed in non-cryptographic hashes - is expected and does not impact SnapRAID's intended use: accidental storage - corruption remains random and is not crafted to exploit the known seed or - internal hash state. - - References: - https://thomasahle.com/blog/adversarial-examples-for-hashes/ - https://thomasahle.com/blog/adversarial-examples-for-hashes/verify/spookyhash2-64/README.html - https://github.com/thomasahle/hash-collision-witnesses/tree/main/spookyhash2-64 - murmur3_snapraid.cpp MurmurHash3 x86_128 as used by SnapRAID, with the SnapRAID 128-bit seed interface. @@ -130,7 +56,7 @@ murmur3_snapraid.cpp MurmurHash3-128-snapraid.txt SMHasher3 output for murmur3_snapraid.cpp. - Result comparison: + Result comparison: The SnapRAID variant passes 179 of 252 tests, compared with 154 of 250 for the official MurmurHash3 x86_128 int32 variant. Most data-path failures are the same, including BIC, Text, TwoBytes, PerlinNoise and diff --git a/tommyds/tommy.c b/tommyds/tommy.c index 51c2859b..f3c16d8c 100644 --- a/tommyds/tommy.c +++ b/tommyds/tommy.c @@ -4,11 +4,10 @@ /* redefine the malloc for tommy use */ #define tommy_malloc malloc_nofail #define tommy_calloc calloc_nofail -#define tommy_realloc realloc_nofail #define tommy_free free #include "os/portable.h" -#include "cmdline/support.h" /* for malloc/calloc/realloc_nofail() */ +#include "cmdline/support.h" /* for malloc/calloc_nofail() */ #include "tommyhash.c" #include "tommyarray.c" diff --git a/tommyds/tommyarray.c b/tommyds/tommyarray.c index 2b9c8261..90445659 100644 --- a/tommyds/tommyarray.c +++ b/tommyds/tommyarray.c @@ -3,18 +3,18 @@ #include "tommyarray.h" -#include /* for memset */ - /******************************************************************************/ /* array */ TOMMY_API void tommy_array_init(tommy_array* array) { + tommy_uint_t i; + /* fixed initial size */ array->bucket_bit = TOMMY_ARRAY_BIT; array->bucket_max = (tommy_size_t)1 << array->bucket_bit; array->bucket[0] = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); - for (tommy_uint_t i = 1; i < TOMMY_ARRAY_BIT; ++i) + for (i = 1; i < TOMMY_ARRAY_BIT; ++i) array->bucket[i] = array->bucket[0]; array->count = 0; @@ -22,18 +22,26 @@ TOMMY_API void tommy_array_init(tommy_array* array) TOMMY_API void tommy_array_done(tommy_array* array) { + tommy_uint_t i; + tommy_free(array->bucket[0]); - for (tommy_uint_t i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { + for (i = TOMMY_ARRAY_BIT; i < array->bucket_bit; ++i) { void** segment = array->bucket[i]; tommy_free(&segment[(tommy_ptrdiff_t)1 << i]); } } -TOMMY_API void tommy_array_reserve(tommy_array* array, tommy_size_t size) +TOMMY_API void tommy_array_grow(tommy_array* array, tommy_size_t count) { - while (size > array->bucket_max) { + if (array->count >= count) + return; + array->count = count; + + while (count > array->bucket_max) { + void** segment; + /* allocate one more segment */ - void** segment = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); + segment = tommy_cast(void**, tommy_calloc(array->bucket_max, sizeof(void*))); /* store it adjusting the offset */ /* cast to ptrdiff_t to ensure to get a negative value */ @@ -44,78 +52,7 @@ TOMMY_API void tommy_array_reserve(tommy_array* array, tommy_size_t size) } } -TOMMY_API void tommy_array_resize(tommy_array* array, tommy_size_t size) -{ - if (size >= array->count) { - tommy_array_grow(array, size); - return; - } - - /* clear the unused elements to maintain the invariant that unused slots are zero */ - tommy_size_t pos = size; - while (pos < array->count) { - tommy_uint_t bsr = tommy_ilog2(pos | 1); - tommy_size_t seg_end = (bsr < TOMMY_ARRAY_BIT) ? ((tommy_size_t)1 << TOMMY_ARRAY_BIT) : ((tommy_size_t)1 << (bsr + 1)); - tommy_size_t chunk_end = array->count < seg_end ? array->count : seg_end; - - memset(&array->bucket[bsr][pos], 0, (chunk_end - pos) * sizeof(void*)); - pos = chunk_end; - } - - array->count = size; -} - -TOMMY_API void tommy_array_shrink(tommy_array* array) -{ - tommy_uint_t target_bucket_bit = TOMMY_ARRAY_BIT; - - if (array->count > (tommy_size_t)1 << TOMMY_ARRAY_BIT) - target_bucket_bit = tommy_ilog2(array->count - 1) + 1; - - while (array->bucket_bit > target_bucket_bit) { - --array->bucket_bit; - void** segment = array->bucket[array->bucket_bit]; - tommy_free(&segment[(tommy_ptrdiff_t)1 << array->bucket_bit]); - } - - array->bucket_max = (tommy_size_t)1 << array->bucket_bit; -} - -TOMMY_API void tommy_array_foreach(tommy_array* array, tommy_foreach_func* func) -{ - tommy_size_t pos = 0; - - while (pos < array->count) { - tommy_uint_t bsr = tommy_ilog2(pos | 1); - tommy_size_t seg_end = (bsr < TOMMY_ARRAY_BIT) ? ((tommy_size_t)1 << TOMMY_ARRAY_BIT) : ((tommy_size_t)1 << (bsr + 1)); - tommy_size_t chunk_end = array->count < seg_end ? array->count : seg_end; - void** ptr = array->bucket[bsr]; - - while (pos < chunk_end) { - func(ptr[pos]); - ++pos; - } - } -} - -TOMMY_API void tommy_array_foreach_arg(tommy_array* array, tommy_foreach_arg_func* func, void* arg) -{ - tommy_size_t pos = 0; - - while (pos < array->count) { - tommy_uint_t bsr = tommy_ilog2(pos | 1); - tommy_size_t seg_end = (bsr < TOMMY_ARRAY_BIT) ? ((tommy_size_t)1 << TOMMY_ARRAY_BIT) : ((tommy_size_t)1 << (bsr + 1)); - tommy_size_t chunk_end = array->count < seg_end ? array->count : seg_end; - void** ptr = array->bucket[bsr]; - - while (pos < chunk_end) { - func(arg, ptr[pos]); - ++pos; - } - } -} - -TOMMY_API tommy_size_t tommy_array_memory_usage(const tommy_array* array) +TOMMY_API tommy_size_t tommy_array_memory_usage(tommy_array* array) { return array->bucket_max * (tommy_size_t)sizeof(void*); } diff --git a/tommyds/tommyarray.h b/tommyds/tommyarray.h index ff92c762..f5f4d726 100644 --- a/tommyds/tommyarray.h +++ b/tommyds/tommyarray.h @@ -7,14 +7,14 @@ * This array is able to grow dynamically upon request, without any reallocation. * * The grow operation involves an allocation of a new array segment, without reallocating - * the already used memory, and thus not increasing the heap fragmentation. - * This also implies that the address of the stored elements never changes. + * the already used memory, and thus **not increasing** the heap fragmentation. + * This also implies that the address of the stored elements never change. * * Allocated segments grow in size exponentially. */ -#ifndef TOMMY_ARRAY_H -#define TOMMY_ARRAY_H +#ifndef __TOMMYARRAY_H +#define __TOMMYARRAY_H #include "tommytypes.h" @@ -42,83 +42,33 @@ typedef struct tommy_array_struct { /** * Initializes the array. - * \param array The array. */ TOMMY_API void tommy_array_init(tommy_array* array); /** * Deinitializes the array. - * \param array The array. */ TOMMY_API void tommy_array_done(tommy_array* array); -/** - * Allocates space for at least the specified number of elements. - * The initialized size and existing elements are unchanged. - * Existing element references remain valid. - * \param array The array. - * \param size Number of elements to reserve space for. - */ -TOMMY_API void tommy_array_reserve(tommy_array* array, tommy_size_t size); - /** * Grows the size up to the specified value. * All the new elements in the array are initialized with the 0 value. - * \param array The array. - * \param size New size of the array. */ -tommy_inline void tommy_array_grow(tommy_array* array, tommy_size_t size) -{ - if (size > array->count) { - array->count = size; - - if (size > array->bucket_max) - tommy_array_reserve(array, size); - } -} +TOMMY_API void tommy_array_grow(tommy_array* array, tommy_size_t size); /** - * Changes the initialized size, preserving the common prefix. - * New elements are initialized to zero. - * Reducing the size does not release allocated capacity. - * \param array The array. - * \param size New size of the array. - */ -TOMMY_API void tommy_array_resize(tommy_array* array, tommy_size_t size); - -/** - * Removes all elements, preserving the allocated capacity. - * Call tommy_array_shrink() after clearing to restore the initial capacity. - * Pointed-to elements are not freed. - * The array remains initialized and can be reused immediately. - * \param array The array. - */ -tommy_inline void tommy_array_clear(tommy_array* array) -{ - tommy_array_resize(array, 0); -} - -/** - * Releases unused allocated memory. - * Preserves the size, values, and addresses of existing elements. - * \param array The array. - */ -TOMMY_API void tommy_array_shrink(tommy_array* array); - -/** - * Gets a reference to the element at the specified position. + * Gets a reference of the element at the specified position. * You must be sure that space for this position is already - * allocated by calling tommy_array_grow(). - * \param array The array. - * \param pos Position of the element. - * \return Pointer to the slot at the specified position. + * allocated calling tommy_array_grow(). */ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) { + tommy_uint_t bsr; + assert(pos < array->count); /* get the highest bit set, in case of all 0, return 0 */ - tommy_uint_t bsr = tommy_ilog2(pos | 1); + bsr = tommy_ilog2(pos | 1); return &array->bucket[bsr][pos]; } @@ -126,10 +76,7 @@ tommy_inline void** tommy_array_ref(tommy_array* array, tommy_size_t pos) /** * Sets the element at the specified position. * You must be sure that space for this position is already - * allocated by calling tommy_array_grow(). - * \param array The array. - * \param pos Position of the element. - * \param element Element to set. + * allocated calling tommy_array_grow(). */ tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* element) { @@ -139,140 +86,36 @@ tommy_inline void tommy_array_set(tommy_array* array, tommy_size_t pos, void* el /** * Gets the element at the specified position. * You must be sure that space for this position is already - * allocated by calling tommy_array_grow(). - * \param array The array. - * \param pos Position of the element. - * \return The element at the specified position. + * allocated calling tommy_array_grow(). */ tommy_inline void* tommy_array_get(tommy_array* array, tommy_size_t pos) { return *tommy_array_ref(array, pos); } -/** - * Gets the last element. - * The array must not be empty. - * \param array The array. - * \return The last element. - */ -tommy_inline void* tommy_array_tail(tommy_array* array) -{ - assert(array->count != 0); - return tommy_array_get(array, array->count - 1); -} - /** * Grows and inserts a new element at the end of the array. - * \param array The array. - * \param element Element to insert. */ -tommy_inline void tommy_array_insert_tail(tommy_array* array, void* element) +tommy_inline void tommy_array_insert(tommy_array* array, void* element) { tommy_size_t pos = array->count; - ++array->count; - if (array->count > array->bucket_max) - tommy_array_reserve(array, array->count); + tommy_array_grow(array, pos + 1); tommy_array_set(array, pos, element); } -/** - * Removes and returns the last element. - * The array must not be empty. - * The removed slot is cleared, and allocated capacity is preserved. - * \param array The array. - * \return The removed element. - */ -tommy_inline void* tommy_array_remove_tail(tommy_array* array) -{ - assert(array->count != 0); - - tommy_size_t pos = array->count - 1; - void** ptr = tommy_array_ref(array, pos); - void* element = *ptr; - - /* keep unused slots zero so growing within capacity needs no initialization */ - *ptr = 0; - array->count = pos; - - return element; -} - -/** - * Checks whether the array is empty. - * \param array The array. - * \return Nonzero if the array is empty, 0 otherwise. - */ -tommy_inline tommy_bool_t tommy_array_empty(const tommy_array* array) -{ - return array->count == 0; -} - /** * Gets the initialized size of the array. - * \param array The array. - * \return The initialized size of the array. */ -tommy_inline tommy_size_t tommy_array_size(const tommy_array* array) +tommy_inline tommy_size_t tommy_array_size(tommy_array* array) { return array->count; } -/** - * Gets the number of elements that fit without further allocation. - * \param array The array. - * \return The capacity of the array. - */ -tommy_inline tommy_size_t tommy_array_capacity(const tommy_array* array) -{ - return array->bucket_max; -} - -/** - * Exchanges two initialized arrays without copying their elements. - * Element references remain valid and belong to the other array. - * Passing the same array twice has no effect. - * \param first The first array. - * \param second The second array. - * \note This operation is O(1). - */ -tommy_inline void tommy_array_swap(tommy_array* first, tommy_array* second) -{ - tommy_array tmp = *first; - *first = *second; - *second = tmp; -} - -/** - * Calls the specified function for each element in the array. - * - * You cannot add or remove elements, nor change the size of the array, - * from inside the callback. - * - * If the callback frees the elements, the array retains dangling - * pointers. Call tommy_array_clear() before reusing it, or - * tommy_array_done() to release its storage. - * \param array The array. - * \param func Function to call with each element. - */ -TOMMY_API void tommy_array_foreach(tommy_array* array, tommy_foreach_func* func); - -/** - * Calls the specified function with an argument for each element in the array. - * - * \param array The array. - * \param func Function to call with each element. - * \param arg Argument to pass to the function. - */ -TOMMY_API void tommy_array_foreach_arg(tommy_array* array, tommy_foreach_arg_func* func, void* arg); - /** * Gets the size of allocated memory. - * \param array The array. - * \return Total memory allocated by the array, in bytes. */ -TOMMY_API tommy_size_t tommy_array_memory_usage(const tommy_array* array); +TOMMY_API tommy_size_t tommy_array_memory_usage(tommy_array* array); #endif - diff --git a/tommyds/tommyarrayblkof.c b/tommyds/tommyarrayblkof.c index a5b1f070..da2d43c6 100644 --- a/tommyds/tommyarrayblkof.c +++ b/tommyds/tommyarrayblkof.c @@ -3,16 +3,12 @@ #include "tommyarrayblkof.h" -#include /* for memset */ - /******************************************************************************/ /* array */ TOMMY_API void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size) { - array->block_capacity = (tommy_size_t)1 << TOMMY_ARRAYBLKOF_BIT; - array->block = tommy_cast(unsigned char**, tommy_malloc(array->block_capacity * sizeof(array->block[0]))); - array->block_count = 0; + tommy_array_init(&array->block); array->element_size = element_size; array->count = 0; @@ -20,127 +16,44 @@ TOMMY_API void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t eleme TOMMY_API void tommy_arrayblkof_done(tommy_arrayblkof* array) { - for (tommy_size_t i = 0; i < array->block_count; ++i) - tommy_free(array->block[i]); + tommy_size_t i; - tommy_free(array->block); + for (i = 0; i < tommy_array_size(&array->block); ++i) + tommy_free(tommy_array_get(&array->block, i)); + + tommy_array_done(&array->block); } -TOMMY_API void tommy_arrayblkof_reserve(tommy_arrayblkof* array, tommy_size_t size) +TOMMY_API void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t count) { - if (size <= array->block_count * TOMMY_ARRAYBLKOF_SIZE) + tommy_size_t block_max; + tommy_size_t block_mac; + + if (array->count >= count) return; + array->count = count; - tommy_size_t block_max = (size - 1) / TOMMY_ARRAYBLKOF_SIZE + 1; + block_max = (count + TOMMY_ARRAYBLKOF_SIZE - 1) / TOMMY_ARRAYBLKOF_SIZE; + block_mac = tommy_array_size(&array->block); - if (array->block_capacity < block_max) { - /* only the directory moves; addresses inside existing blocks stay valid */ - tommy_size_t capacity = array->block_capacity; - while (capacity < block_max) - capacity *= 2; - array->block = tommy_cast(unsigned char**, tommy_realloc(array->block, capacity * sizeof(array->block[0]))); - array->block_capacity = capacity; - } + if (block_mac < block_max) { + /* grow the block array */ + tommy_array_grow(&array->block, block_max); - /* allocate new blocks */ - while (array->block_count < block_max) { - unsigned char* ptr = tommy_cast(unsigned char*, tommy_calloc(TOMMY_ARRAYBLKOF_SIZE, array->element_size)); + /* allocate new blocks */ + while (block_mac < block_max) { + void** ptr = tommy_cast(void**, tommy_calloc(TOMMY_ARRAYBLKOF_SIZE, array->element_size)); - array->block[array->block_count] = ptr; - ++array->block_count; - } -} + /* set the new block */ + tommy_array_set(&array->block, block_mac, ptr); -TOMMY_API void tommy_arrayblkof_resize(tommy_arrayblkof* array, tommy_size_t size) -{ - if (size >= array->count) { - tommy_arrayblkof_grow(array, size); - return; - } - - /* clear the unused elements to maintain the invariant that unused slots are zero */ - tommy_size_t pos = size; - while (pos < array->count) { - tommy_size_t blk_idx = pos / TOMMY_ARRAYBLKOF_SIZE; - tommy_size_t blk_offset = pos % TOMMY_ARRAYBLKOF_SIZE; - tommy_size_t blk_rem = TOMMY_ARRAYBLKOF_SIZE - blk_offset; - tommy_size_t count_rem = array->count - pos; - tommy_size_t chunk = blk_rem < count_rem ? blk_rem : count_rem; - - memset(array->block[blk_idx] + blk_offset * array->element_size, 0, chunk * array->element_size); - pos += chunk; - } - - array->count = size; -} - -TOMMY_API void tommy_arrayblkof_shrink(tommy_arrayblkof* array) -{ - tommy_size_t block_max = (array->count == 0) ? 0 : ((array->count - 1) / TOMMY_ARRAYBLKOF_SIZE + 1); - - while (array->block_count > block_max) { - --array->block_count; - tommy_free(array->block[array->block_count]); - } - - tommy_size_t min_capacity = (tommy_size_t)1 << TOMMY_ARRAYBLKOF_BIT; - tommy_size_t capacity = min_capacity; - while (capacity < block_max) - capacity *= 2; - if (capacity < array->block_capacity) { - array->block = tommy_cast(unsigned char**, tommy_realloc(array->block, capacity * sizeof(array->block[0]))); - array->block_capacity = capacity; - } -} - -TOMMY_API void tommy_arrayblkof_foreach(tommy_arrayblkof* array, tommy_foreach_func* func) -{ - /* cache the element size to avoid reloading it after each callback */ - tommy_size_t element_size = array->element_size; - tommy_size_t pos = 0; - - while (pos < array->count) { - tommy_size_t blk_idx = pos / TOMMY_ARRAYBLKOF_SIZE; - tommy_size_t chunk = array->count - pos; - unsigned char* ptr = array->block[blk_idx]; - - if (chunk > TOMMY_ARRAYBLKOF_SIZE) - chunk = TOMMY_ARRAYBLKOF_SIZE; - - for (tommy_size_t i = 0; i < chunk; ++i) { - func(ptr); - ptr += element_size; + ++block_mac; } - - pos += chunk; } } -TOMMY_API void tommy_arrayblkof_foreach_arg(tommy_arrayblkof* array, tommy_foreach_arg_func* func, void* arg) +TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array) { - /* cache the element size to avoid reloading it after each callback */ - tommy_size_t element_size = array->element_size; - tommy_size_t pos = 0; - - while (pos < array->count) { - tommy_size_t blk_idx = pos / TOMMY_ARRAYBLKOF_SIZE; - tommy_size_t chunk = array->count - pos; - unsigned char* ptr = array->block[blk_idx]; - - if (chunk > TOMMY_ARRAYBLKOF_SIZE) - chunk = TOMMY_ARRAYBLKOF_SIZE; - - for (tommy_size_t i = 0; i < chunk; ++i) { - func(arg, ptr); - ptr += element_size; - } - - pos += chunk; - } -} - -TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(const tommy_arrayblkof* array) -{ - return array->block_capacity * sizeof(array->block[0]) + array->block_count * TOMMY_ARRAYBLKOF_SIZE * array->element_size; + return tommy_array_memory_usage(&array->block) + tommy_array_size(&array->block) * TOMMY_ARRAYBLKOF_SIZE * array->element_size; } diff --git a/tommyds/tommyarrayblkof.h b/tommyds/tommyarrayblkof.h index 7178ae42..8c9edfa0 100644 --- a/tommyds/tommyarrayblkof.h +++ b/tommyds/tommyarrayblkof.h @@ -2,43 +2,29 @@ // Copyright (C) 2013 Andrea Mazzoleni /** \file - * Dynamic array based on fixed-size blocks. + * Dynamic array based on blocks of fixed size. * - * This array is able to grow dynamically without reallocating stored elements. - * The directory of block pointers may be reallocated, but element addresses - * never change. + * This array is able to grow dynamically upon request, without any reallocation. * - * This is very similar to ::tommy_arrayblk, but it allows storing elements of any + * This is very similar to ::tommy_arrayblk, but it allows to store elements of any * size and not just pointers. * - * The container allocates space for elements but never copies user data into - * or out of it. tommy_arrayblkof_ref(), tommy_arrayblkof_tail(), and - * tommy_arrayblkof_insert_tail() return an element's address so the caller can - * read or write it directly. - * - * \note Only types with fundamental alignment requirements are supported. - * Storage is allocated using calloc(), which does not guarantee the alignment - * required by over-aligned types (e.g. _Alignas(32) or _Alignas(64)). - * When storing typed elements, use sizeof(type) as the element size to - * preserve alignment for every element. + * Note that in this case tommy_arrayblkof_ref() returns a pointer to the element, + * that should be used for getting and setting elements in the array, + * as generic getter and setter are not available. */ -#ifndef TOMMY_ARRAYBLKOF_H -#define TOMMY_ARRAYBLKOF_H +#ifndef __TOMMYARRAYBLKOF_H +#define __TOMMYARRAYBLKOF_H #include "tommytypes.h" +#include "tommyarray.h" #include /* for assert */ -#include /* for memset */ /******************************************************************************/ /* array */ -/** - * Initial and minimal capacity of the block directory expressed as a power of 2. - */ -#define TOMMY_ARRAYBLKOF_BIT 6 - /** * Elements for each block. */ @@ -49,214 +35,55 @@ * \note Don't use internal fields directly, but access the container only using functions. */ typedef struct tommy_arrayblkof_struct { - unsigned char** block; /**< Directory of blocks. */ - tommy_size_t block_count; /**< Number of allocated blocks. */ - tommy_size_t block_capacity; /**< Number of directory entries allocated. */ + tommy_array block; /**< Array of blocks. */ tommy_size_t element_size; /**< Size of the stored element in bytes. */ tommy_size_t count; /**< Number of initialized elements in the array. */ } tommy_arrayblkof; /** * Initializes the array. - * \param array The array. - * \param element_size Size in bytes of the element to store in the array. + * \param element_size Size in byte of the element to store in the array. */ TOMMY_API void tommy_arrayblkof_init(tommy_arrayblkof* array, tommy_size_t element_size); /** * Deinitializes the array. - * \param array The array. */ TOMMY_API void tommy_arrayblkof_done(tommy_arrayblkof* array); -/** - * Allocates space for at least the specified number of elements. - * The initialized size and existing elements are unchanged. - * Existing element references remain valid. - * \param array The array. - * \param size Number of elements to reserve space for. - */ -TOMMY_API void tommy_arrayblkof_reserve(tommy_arrayblkof* array, tommy_size_t size); - /** * Grows the size up to the specified value. * All the new elements in the array are initialized with the 0 value. - * \param array The array. - * \param size New size of the array. */ -tommy_inline void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size) -{ - if (size > array->count) { - array->count = size; - - if (size > array->block_count * TOMMY_ARRAYBLKOF_SIZE) - tommy_arrayblkof_reserve(array, size); - } -} +TOMMY_API void tommy_arrayblkof_grow(tommy_arrayblkof* array, tommy_size_t size); /** - * Changes the initialized size, preserving the common prefix. - * New elements are initialized to zero. - * Reducing the size does not release allocated capacity. - * \param array The array. - * \param size New size of the array. - */ -TOMMY_API void tommy_arrayblkof_resize(tommy_arrayblkof* array, tommy_size_t size); - -/** - * Removes all elements, preserving the allocated capacity. - * Call tommy_arrayblkof_shrink() after clearing to restore the initial capacity. - * The array remains initialized and can be reused immediately. - * \param array The array. - */ -tommy_inline void tommy_arrayblkof_clear(tommy_arrayblkof* array) -{ - tommy_arrayblkof_resize(array, 0); -} - -/** - * Releases unused allocated memory. - * Preserves the size, values, and addresses of existing elements. - * \param array The array. - */ -TOMMY_API void tommy_arrayblkof_shrink(tommy_arrayblkof* array); - -/** - * Gets a reference to the element at the specified position. + * Gets a reference of the element at the specified position. * You must be sure that space for this position is already - * allocated by calling tommy_arrayblkof_grow(). - * \param array The array. - * \param pos Position of the element. - * \return Pointer to the slot at the specified position. + * allocated calling tommy_arrayblkof_grow(). */ tommy_inline void* tommy_arrayblkof_ref(tommy_arrayblkof* array, tommy_size_t pos) { + unsigned char* base; + assert(pos < array->count); - return array->block[pos / TOMMY_ARRAYBLKOF_SIZE] + (pos % TOMMY_ARRAYBLKOF_SIZE) * array->element_size; -} + base = tommy_cast(unsigned char*, tommy_array_get(&array->block, pos / TOMMY_ARRAYBLKOF_SIZE)); -/** - * Gets a reference to the last element. - * The array must not be empty. - * \param array The array. - * \return Pointer to the last element. - */ -tommy_inline void* tommy_arrayblkof_tail(tommy_arrayblkof* array) -{ - assert(array->count != 0); - return tommy_arrayblkof_ref(array, array->count - 1); -} - -/** - * Adds a zero-initialized element and returns its reference. - * \param array The array. - * \return Pointer to the new element. - */ -tommy_inline void* tommy_arrayblkof_insert_tail(tommy_arrayblkof* array) -{ - tommy_size_t pos = array->count; - - ++array->count; - if (array->count > array->block_count * TOMMY_ARRAYBLKOF_SIZE) - tommy_arrayblkof_reserve(array, array->count); - - return tommy_arrayblkof_ref(array, pos); -} - -/** - * Removes the last element without copying it. - * The array must not be empty. The removed slot is cleared. - * \param array The array. - */ -tommy_inline void tommy_arrayblkof_remove_tail(tommy_arrayblkof* array) -{ - assert(array->count != 0); - - tommy_size_t pos = array->count - 1; - void* ptr = tommy_arrayblkof_ref(array, pos); - - /* keep unused slots zero so growing within capacity needs no initialization */ - memset(ptr, 0, array->element_size); - array->count = pos; -} - -/** - * Checks whether the array is empty. - * \param array The array. - * \return Nonzero if the array is empty, 0 otherwise. - */ -tommy_inline tommy_bool_t tommy_arrayblkof_empty(const tommy_arrayblkof* array) -{ - return array->count == 0; + return base + (pos % TOMMY_ARRAYBLKOF_SIZE) * array->element_size; } /** * Gets the initialized size of the array. - * \param array The array. - * \return The initialized size of the array. */ -tommy_inline tommy_size_t tommy_arrayblkof_size(const tommy_arrayblkof* array) +tommy_inline tommy_size_t tommy_arrayblkof_size(tommy_arrayblkof* array) { return array->count; } -/** - * Gets the number of elements that fit without further allocation. - * \param array The array. - * \return The capacity of the array. - */ -tommy_inline tommy_size_t tommy_arrayblkof_capacity(const tommy_arrayblkof* array) -{ - return array->block_count * TOMMY_ARRAYBLKOF_SIZE; -} - -/** - * Exchanges two initialized arrays without copying their elements. - * Element references remain valid and belong to the other array. - * Passing the same array twice has no effect. - * \param first The first array. - * \param second The second array. - * \note This operation is O(1). - */ -tommy_inline void tommy_arrayblkof_swap(tommy_arrayblkof* first, tommy_arrayblkof* second) -{ - /* keep record size with the storage it describes */ - tommy_arrayblkof tmp = *first; - *first = *second; - *second = tmp; -} - -/** - * Calls the specified function for each element in the array. - * - * You cannot add or remove elements, nor change the size of the array, - * from inside the callback. - * - * Elements are stored inside the array: passing free() as the callback - * is invalid. A callback may release resources owned by each element, - * but must not free the element itself. Call tommy_arrayblkof_clear() before - * reusing it, or tommy_arrayblkof_done() to release its storage. - * \param array The array. - * \param func Function to call with each element. - */ -TOMMY_API void tommy_arrayblkof_foreach(tommy_arrayblkof* array, tommy_foreach_func* func); - -/** - * Calls the specified function with an argument for each element in the array. - * - * \param array The array. - * \param func Function to call with each element. - * \param arg Argument to pass to the function. - */ -TOMMY_API void tommy_arrayblkof_foreach_arg(tommy_arrayblkof* array, tommy_foreach_arg_func* func, void* arg); - /** * Gets the size of allocated memory. - * \param array The array. - * \return Total memory allocated by the array, in bytes. */ -TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(const tommy_arrayblkof* array); +TOMMY_API tommy_size_t tommy_arrayblkof_memory_usage(tommy_arrayblkof* array); #endif - diff --git a/tommyds/tommychain.h b/tommyds/tommychain.h index d1c293f6..c864dccd 100644 --- a/tommyds/tommychain.h +++ b/tommyds/tommychain.h @@ -9,8 +9,8 @@ * Do not use this directly. Use lists instead. */ -#ifndef TOMMY_CHAIN_H -#define TOMMY_CHAIN_H +#ifndef __TOMMYCHAIN_H +#define __TOMMYCHAIN_H #include "tommytypes.h" @@ -65,39 +65,31 @@ tommy_inline void tommy_chain_concat(tommy_node* first_tail, tommy_node* second_ /** * Merges two chains. * \param first First chain (will contain the result). - * \param second Second chain (consumed by the merge; its descriptor is not cleared - * and must not be used as an independent chain afterward). + * \param second Second chain (will be empty after the merge). * \param cmp Comparison function. */ -tommy_inline void tommy_chain_merge(tommy_chain* tommy_restrict first, tommy_chain* tommy_restrict second, tommy_compare_func* cmp) +tommy_inline void tommy_chain_merge(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) { - /* - * comparators must not change the chains, so cache these endpoints across calls. - * also caching first_tail caused register spills with GCC and Clang on x86-64. - */ - tommy_node* first_head = first->head; - tommy_node* second_tail = second->tail; - tommy_node* first_i = first_head; + tommy_node* first_i = first->head; tommy_node* second_i = second->head; /* merge */ while (1) { if (cmp(first_i->data, second_i->data) > 0) { tommy_node* next = second_i->next; - if (first_i == first_head) { + if (first_i == first->head) { tommy_chain_concat(second_i, first_i); - first_head = second_i; - first->head = first_head; + first->head = second_i; } else { tommy_chain_splice(first_i->prev, first_i, second_i, second_i); } - if (second_i == second_tail) + if (second_i == second->tail) break; second_i = next; } else { if (first_i == first->tail) { tommy_chain_concat(first_i, second_i); - first->tail = second_tail; + first->tail = second->tail; break; } first_i = first_i->next; @@ -106,36 +98,13 @@ tommy_inline void tommy_chain_merge(tommy_chain* tommy_restrict first, tommy_cha } /** - * Merges two chains containing one node each. - * \param first First singleton chain (will contain the result). - * \param second Second singleton chain (consumed by the merge; its descriptor is not cleared - * and must not be used as an independent chain afterward). - * \param cmp Comparison function. - */ -tommy_inline void tommy_chain_merge_singleton(tommy_chain* tommy_restrict first, tommy_chain* tommy_restrict second, tommy_compare_func* cmp) -{ - tommy_node* first_node = first->head; - tommy_node* second_node = second->head; - - /* keep equal elements in their original order for a stable sort */ - if (cmp(first_node->data, second_node->data) <= 0) { - tommy_chain_concat(first_node, second_node); - first->tail = second_node; - } else { - tommy_chain_concat(second_node, first_node); - first->head = second_node; - } -} - -/** - * Merges two chains managing special degenerate cases. + * Merges two chains managing special degenerated cases. * It's functionally equivalent to tommy_chain_merge() but faster with already ordered chains. * \param first First chain (will contain the result). - * \param second Second chain (consumed by the merge; its descriptor is not cleared - * and must not be used as an independent chain afterward). + * \param second Second chain (will be empty after the merge). * \param cmp Comparison function. */ -tommy_inline void tommy_chain_merge_degenerated(tommy_chain* tommy_restrict first, tommy_chain* tommy_restrict second, tommy_compare_func* cmp) +tommy_inline void tommy_chain_merge_degenerated(tommy_chain* first, tommy_chain* second, tommy_compare_func* cmp) { /* identify the condition first <= second */ if (cmp(first->tail->data, second->head->data) <= 0) { @@ -157,9 +126,9 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* tommy_restrict firs /** * Sorts a chain. - * It's a stable merge sort using power of 2 buckets, with O(n log n) complexity, + * It's a stable merge sort using power of 2 buckets, with O(N*log(N)) complexity, * similar to the one used in the SGI STL libraries and in the Linux Kernel, - * but faster on degenerate cases like already ordered lists. + * but faster on degenerated cases like already ordered lists. * * SGI STL stl_list.h * http://www.sgi.com/tech/stl/stl_list.h @@ -167,48 +136,43 @@ tommy_inline void tommy_chain_merge_degenerated(tommy_chain* tommy_restrict firs * Linux Kernel lib/list_sort.c * http://lxr.linux.no/#linux+v2.6.36/lib/list_sort.c * - * \param chain The chain. + * \param chain Chain to sort. * \param cmp Comparison function. */ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* cmp) { /* * Bit buckets of chains. - * Each bucket contains 2^(i+1) nodes or it's empty. - * The bucket containing an unpaired final node has one node less. + * Each bucket contains 2^i nodes or it's empty. * The chain at address TOMMY_BIT_MAX is an independent variable operating as "carry". * We keep it in the same "bit" vector to avoid reports from the valgrind tool sgcheck. */ tommy_chain bit[TOMMY_SIZE_BIT + 1]; - /* + /** * Value stored inside the bit bucket. * It's used to know which bucket is empty or full. */ - tommy_size_t counter = 0; + tommy_size_t counter; tommy_node* node = chain->head; tommy_node* tail = chain->tail; + tommy_size_t mask; + tommy_size_t i; + counter = 0; while (1) { + tommy_node* next; + tommy_chain* last; + /* carry bit to add */ - tommy_chain* last = &bit[TOMMY_SIZE_BIT]; + last = &bit[TOMMY_SIZE_BIT]; bit[TOMMY_SIZE_BIT].head = node; bit[TOMMY_SIZE_BIT].tail = node; - tommy_node* next = node->next; - - /* start with a sorted pair to halve the number of bucket insertions */ - if (node != tail) { - node = next; - /* save the next input node before merging can change the links */ - next = node->next; - tommy_chain second = { node, node }; - tommy_chain_merge_singleton(last, &second, cmp); - } + next = node->next; /* add the bit, propagating the carry */ - tommy_size_t i = 0; - tommy_size_t mask = counter; - + i = 0; + mask = counter; while ((mask & 1) != 0) { tommy_chain_merge_degenerated(&bit[i], last, cmp); mask >>= 1; @@ -228,8 +192,8 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* } /* merge the buckets */ - tommy_size_t i = tommy_ctz(counter); - tommy_size_t mask = counter >> i; + i = tommy_ctz(counter); + mask = counter >> i; while (mask != 1) { mask >>= 1; if (mask & 1) @@ -243,4 +207,3 @@ tommy_inline void tommy_chain_mergesort(tommy_chain* chain, tommy_compare_func* } #endif - diff --git a/tommyds/tommyhash.c b/tommyds/tommyhash.c index fc4fd276..f6b022b3 100644 --- a/tommyds/tommyhash.c +++ b/tommyds/tommyhash.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: BSD-2-Clause AND MIT +// SPDX-License-Identifier: BSD-2-Clause // Copyright (C) 2010 Andrea Mazzoleni #include "tommyhash.h" @@ -13,178 +13,222 @@ * They are needed only if BigEndian. */ #if defined(__GNUC__) -#define tommy_imp_hash_swap32(x) __builtin_bswap32(x) +#define tommy_swap32(x) __builtin_bswap32(x) #else -tommy_inline tommy_uint32_t tommy_imp_hash_swap32(tommy_uint32_t v) +tommy_inline tommy_uint32_t tommy_swap32(tommy_uint32_t v) { return ((v & 0xFF000000) >> 24) | - ((v & 0x00FF0000) >> 8) | - ((v & 0x0000FF00) << 8) | - ((v & 0x000000FF) << 24); + ((v & 0x00FF0000) >> 8) | + ((v & 0x0000FF00) << 8) | + ((v & 0x000000FF) << 24); } #endif -tommy_inline tommy_uint32_t tommy_imp_hash_le_uint32_read(const void* ptr) +tommy_inline tommy_uint32_t tommy_le_uint32_read(const void* ptr) { tommy_uint32_t v; memcpy(&v, ptr, sizeof(v)); #if defined(WORDS_BIGENDIAN) || defined(__BIG_ENDIAN__) || \ (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) - v = tommy_imp_hash_swap32(v); + v = tommy_swap32(v); #endif return v; } -/******************************************************************************/ -/* a5hash */ +#define tommy_rot(x, k) \ + (((x) << (k)) | ((x) >> (32 - (k)))) -/** - * a5hash32 and a5hash (64-bit) by Aleksey Vaneev, from https://github.com/avaneev/a5hash. - * Adapted from version 5.25 with little-endian word reads. - * The 32-bit variant additionally mixes the expanded seed asymmetrically into - * h[2] and h[3] before processing the input and produces different hash values. - * Copyright (c) 2025 Aleksey Vaneev. Released under the MIT license, see LICENSES/MIT. - */ +#define tommy_mix(a, b, c) \ + do { \ + a -= c; a ^= tommy_rot(c, 4); c += b; \ + b -= a; b ^= tommy_rot(a, 6); a += c; \ + c -= b; c ^= tommy_rot(b, 8); b += a; \ + a -= c; a ^= tommy_rot(c, 16); c += b; \ + b -= a; b ^= tommy_rot(a, 19); a += c; \ + c -= b; c ^= tommy_rot(b, 4); b += a; \ + } while (0) -tommy_inline void tommy_mul(tommy_uint32_t* lo, tommy_uint32_t* hi) -{ - /* both operands are 32-bit; the full product supplies two state words */ - tommy_uint64_t v = (tommy_uint64_t)*lo * *hi; - *lo = (tommy_uint32_t)v; - *hi = v >> 32; -} +#define tommy_final(a, b, c) \ + do { \ + c ^= b; c -= tommy_rot(b, 14); \ + a ^= c; a -= tommy_rot(c, 11); \ + b ^= a; b -= tommy_rot(a, 25); \ + c ^= b; c -= tommy_rot(b, 16); \ + a ^= c; a -= tommy_rot(c, 4); \ + b ^= a; b -= tommy_rot(a, 14); \ + c ^= b; c -= tommy_rot(b, 24); \ + } while (0) TOMMY_API tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len) { const unsigned char* key = tommy_cast(const unsigned char*, void_key); - tommy_uint32_t val01 = 0x55555555; - tommy_uint32_t val10 = 0xaaaaaaaa; - tommy_uint32_t h[4] = { 0x243f6a88, 0x85a308d3, 0xfb0bd3ea, 0x0f58fd47 }; - tommy_uint32_t a, b; - tommy_size_t len = key_len; + tommy_uint32_t a, b, c; - h[0] ^= key_len; - h[1] ^= key_len; -#if TOMMY_SIZE_BIT == 64 - h[2] = (key_len >> 32) ^ 0x452821e6; - h[3] = (key_len >> 32) ^ 0x38d01377; - tommy_mul(&h[2], &h[3]); -#endif - h[0] ^= init_val & val01; - h[1] ^= init_val & val10; - tommy_mul(&h[0], &h[1]); + a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + init_val; - /* - * Mix the expanded seed asymmetrically into h[2] and h[3] so that short keys - * skipping the main loop still depend on the seed, and neither h[0] nor h[1] - * cancels out when folding h[0] ^= h[2] and h[1] ^= h[3] at the end. - */ - h[2] ^= h[0] ^ h[1]; - h[3] ^= h[0]; + while (key_len > 12) { + a += tommy_le_uint32_read(key + 0); + b += tommy_le_uint32_read(key + 4); + c += tommy_le_uint32_read(key + 8); - if (len <= 16) { - if (len >= 4) { - const unsigned char* tail = key + len - 4; - a = tommy_imp_hash_le_uint32_read(key); - b = tommy_imp_hash_le_uint32_read(tail); - if (len >= 9) { - tommy_size_t offset = (len / 8) * 4; - h[2] += tommy_imp_hash_le_uint32_read(key + offset); - h[3] += tommy_imp_hash_le_uint32_read(tail - offset); - tommy_mul(&h[2], &h[3]); - } - } else { - a = 0; - b = 0; - if (len != 0) { - a = key[0]; - if (len >= 2) - a |= key[1] << 8; - if (len == 3) - a |= key[2] << 16; - } - } - } else { - val01 ^= h[0]; - val10 ^= h[1]; - do { - tommy_uint32_t s1 = h[0]; - tommy_uint32_t s4 = h[3]; - h[0] += tommy_imp_hash_le_uint32_read(key); - h[1] += tommy_imp_hash_le_uint32_read(key + 4); - tommy_mul(&h[0], &h[1]); - h[2] += tommy_imp_hash_le_uint32_read(key + 8); - h[3] += tommy_imp_hash_le_uint32_read(key + 12); - tommy_mul(&h[2], &h[3]); - h[0] += val01; - h[1] += s4; - h[2] += s1; - h[3] += val10; - key += 16; - len -= 16; - } while (len > 16); + tommy_mix(a, b, c); - /* the last words may overlap the previous block, but stay within the input */ - const unsigned char* tail = key + len; - a = tommy_imp_hash_le_uint32_read(tail - 8); - b = tommy_imp_hash_le_uint32_read(tail - 4); - if (len >= 9) { - h[2] += tommy_imp_hash_le_uint32_read(tail - 16); - h[3] += tommy_imp_hash_le_uint32_read(tail - 12); - tommy_mul(&h[2], &h[3]); - } + key_len -= 12; + key += 12; } - h[0] ^= h[2]; - h[1] ^= h[3]; - h[0] += a; - h[1] += b; - tommy_mul(&h[0], &h[1]); - h[0] ^= val01; - tommy_mul(&h[0], &h[1]); - return h[0] ^ h[1]; + switch (key_len) { + case 0 : + return c; /* used only when called with a zero length */ + case 12 : + c += tommy_le_uint32_read(key + 8); + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ + case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ + case 9 : c += key[8]; /* fallthrough */ + case 8 : + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ + case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ + case 5 : b += key[4]; /* fallthrough */ + case 4 : + a += tommy_le_uint32_read(key + 0); + break; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ + case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ + case 1 : a += key[0]; /* fallthrough */ + } + + tommy_final(a, b, c); + + return c; } TOMMY_API tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len) { const unsigned char* key = tommy_cast(const unsigned char*, void_key); - tommy_uint64_t val01 = 0x5555555555555555ULL; - tommy_uint64_t val10 = 0xaaaaaaaaaaaaaaaaULL; - tommy_uint64_t h[2] = { (0x452821e638d01377ULL ^ key_len) ^ (init_val & val10), (0x243f6a8885a308d3ULL ^ key_len) ^ (init_val & val01) }; - tommy_size_t len = key_len; + tommy_uint32_t a, b, c; - tommy_mum(&h[0], &h[1]); - if (len > 16) { - val01 ^= h[0]; - val10 ^= h[1]; - do { - /* each 64-bit operand puts the first little-endian word in its high half */ - h[0] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key) << 32 ^ tommy_imp_hash_le_uint32_read(key + 4); - h[1] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key + 8) << 32 ^ tommy_imp_hash_le_uint32_read(key + 12); - tommy_mum(&h[0], &h[1]); - h[0] += val01; - h[1] += val10; - key += 16; - len -= 16; - } while (len > 16); + a = b = c = 0xdeadbeef + ((tommy_uint32_t)key_len) + (init_val & 0xffffffff); + c += init_val >> 32; + + while (key_len > 12) { + a += tommy_le_uint32_read(key + 0); + b += tommy_le_uint32_read(key + 4); + c += tommy_le_uint32_read(key + 8); + + tommy_mix(a, b, c); + + key_len -= 12; + key += 12; } - if (len >= 4) { - const unsigned char* tail = key + len - 4; - tommy_size_t offset = (len / 8) * 4; - h[0] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key) << 32 | tommy_imp_hash_le_uint32_read(tail); - h[1] ^= (tommy_uint64_t)tommy_imp_hash_le_uint32_read(key + offset) << 32 | tommy_imp_hash_le_uint32_read(tail - offset); - } else if (len != 0) { - h[0] ^= key[0]; - if (len >= 2) - h[0] ^= key[1] << 8; - if (len == 3) - h[0] ^= key[2] << 16; + switch (key_len) { + case 0 : + return c + ((tommy_uint64_t)b << 32); /* used only when called with a zero length */ + case 12 : + c += tommy_le_uint32_read(key + 8); + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 11 : c += ((tommy_uint32_t)key[10]) << 16; /* fallthrough */ + case 10 : c += ((tommy_uint32_t)key[9]) << 8; /* fallthrough */ + case 9 : c += key[8]; /* fallthrough */ + case 8 : + b += tommy_le_uint32_read(key + 4); + a += tommy_le_uint32_read(key + 0); + break; + case 7 : b += ((tommy_uint32_t)key[6]) << 16; /* fallthrough */ + case 6 : b += ((tommy_uint32_t)key[5]) << 8; /* fallthrough */ + case 5 : b += key[4]; /* fallthrough */ + case 4 : + a += tommy_le_uint32_read(key + 0); + break; + case 3 : a += ((tommy_uint32_t)key[2]) << 16; /* fallthrough */ + case 2 : a += ((tommy_uint32_t)key[1]) << 8; /* fallthrough */ + case 1 : a += key[0]; /* fallthrough */ } - tommy_mum(&h[0], &h[1]); - h[0] ^= val01; - tommy_mum(&h[0], &h[1]); - return h[0] ^ h[1]; + tommy_final(a, b, c); + + return c + ((tommy_uint64_t)b << 32); +} + +TOMMY_API tommy_uint32_t tommy_strhash_u32(tommy_uint32_t init_val, const void* void_key) +{ + const unsigned char* key = tommy_cast(const unsigned char*, void_key); + tommy_uint32_t a, b, c; + tommy_uint32_t m[3] = { 0xff, 0xff00, 0xff0000 }; + + a = b = c = 0xdeadbeef + init_val; + /* this is different than original lookup3 and the result won't match */ + + while (1) { + tommy_uint32_t v = tommy_le_uint32_read(key); + + if (tommy_haszero_u32(v)) { + if (v & m[0]) { + a += v & m[0]; + if (v & m[1]) { + a += v & m[1]; + if (v & m[2]) + a += v & m[2]; + } + } + + break; + } + + a += v; + + v = tommy_le_uint32_read(key + 4); + + if (tommy_haszero_u32(v)) { + if (v & m[0]) { + b += v & m[0]; + if (v & m[1]) { + b += v & m[1]; + if (v & m[2]) + b += v & m[2]; + } + } + + break; + } + + b += v; + + v = tommy_le_uint32_read(key + 8); + + if (tommy_haszero_u32(v)) { + if (v & m[0]) { + c += v & m[0]; + if (v & m[1]) { + c += v & m[1]; + if (v & m[2]) + c += v & m[2]; + } + } + + break; + } + + c += v; + + tommy_mix(a, b, c); + + key += 12; + } + + /* for lengths that are multipliers of 12 we already have called mix */ + /* this is different than the original lookup3 and the result won't match */ + + tommy_final(a, b, c); + + return c; } diff --git a/tommyds/tommyhash.h b/tommyds/tommyhash.h index a0dd6043..763f5a48 100644 --- a/tommyds/tommyhash.h +++ b/tommyds/tommyhash.h @@ -2,145 +2,79 @@ // Copyright (C) 2010 Andrea Mazzoleni /** \file - * Hash functions for use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. - * - * This module provides hash functions designed for in-memory data structures. - * ::tommy_hash_u32 and ::tommy_strhash_u32 use a modified a5hash32, while - * ::tommy_hash_u64 and ::tommy_strhash_u64 use a5hash (64-bit). - * Reversible integer hash functions are also provided for 32-bit and 64-bit keys. - * - * a5hash32 is intended for 32-bit platforms and uses 32-bit operands with - * full 64-bit multiplication results. a5hash uses 64-bit operands with - * full 128-bit multiplication results. Both algorithms pass SMHasher3 tests. - * - * Prefer the 64-bit version whenever possible, as it provides better resistance - * to collisions, particularly with specially crafted inputs. However, neither - * version is designed to resist deliberate collision attacks. If resistance to - * such attacks is required, use a dedicated hash function designed for that purpose. - * - * All hash functions are intended for runtime in-memory use only and not - * for persistent storage, as hash compatibility across different versions of the - * library is not guaranteed. String and sized buffer hash functions produce - * identical results for 0-terminated strings. + * Hash functions for the use with ::tommy_hashtable, ::tommy_hashdyn and ::tommy_hashlin. */ -#ifndef TOMMY_HASH_H -#define TOMMY_HASH_H +#ifndef __TOMMYHASH_H +#define __TOMMYHASH_H #include "tommytypes.h" -#include /* for strlen */ /******************************************************************************/ /* hash */ /** - * Hash function with a 32-bit result. - * Modified implementation of a5hash32 by Aleksey Vaneev, - * from https://github.com/avaneev/a5hash + * Hash function with a 32 bits result. + * Implementation of the Robert Jenkins "lookup3" hash 32 bits version, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). * - * The expanded seed is mixed asymmetrically into the two additional state words - * before processing the input. This preserves seed dependence for short keys - * that skip the main loop and avoids cancellation of either seed state word - * in the final XOR fold. The resulting hash values differ from upstream a5hash32. - * - * This algorithm is compact, fast on short inputs, and passes SMHasher3 tests. + * This hash is designed to provide a good overall performance on all platforms, + * including 32 bits. If you target only 64 bits, you can use faster hashes, + * like SpookyHash or FarmHash. * * \param init_val Initialization value. * Using a different initialization value, you can generate a completely different set of hash values. * Use 0 if not relevant. - * \param void_key Pointer to the data to hash. Must not be 0, even if key_len is 0. + * \param void_key Pointer to the data to hash. * \param key_len Size of the data to hash. - * \return The 32-bit hash value. * \note - * This function is endianness independent and alignment independent. - * It produces identical hash results to ::tommy_strhash_u32 for 0-terminated strings. - * Compatibility across different versions of the library is not guaranteed: - * this hash is intended for in-memory runtime use only and not for persistent storage. - * It is non-cryptographic and is not designed to protect against malicious collision attacks. + * This function is endianness independent. + * \return The hash value of 32 bits. */ TOMMY_API tommy_uint32_t tommy_hash_u32(tommy_uint32_t init_val, const void* void_key, tommy_size_t key_len); /** - * Hash function with a 64-bit result. - * Implementation of a5hash (64-bit) by Aleksey Vaneev, - * from https://github.com/avaneev/a5hash + * Hash function with a 64 bits result. + * Implementation of the Robert Jenkins "lookup3" hash 64 bits version, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle2(). * - * This algorithm is compact, fast on short inputs, and passes SMHasher3 tests. + * This hash is designed to provide a good overall performance on all platforms, + * including 32 bits. If you target only 64 bits, you can use faster hashes, + * like SpookyHash or FarmHash. * * \param init_val Initialization value. * Using a different initialization value, you can generate a completely different set of hash values. * Use 0 if not relevant. - * \param void_key Pointer to the data to hash. Must not be 0, even if key_len is 0. + * \param void_key Pointer to the data to hash. * \param key_len Size of the data to hash. - * \return The 64-bit hash value. * \note - * This function is endianness independent and alignment independent. - * It produces identical hash results to ::tommy_strhash_u64 for 0-terminated strings. - * Compatibility across different versions of the library is not guaranteed: - * this hash is intended for in-memory runtime use only and not for persistent storage. - * It is non-cryptographic and is not designed to protect against malicious collision attacks. + * This function is endianness independent. + * \return The hash value of 64 bits. */ TOMMY_API tommy_uint64_t tommy_hash_u64(tommy_uint64_t init_val, const void* void_key, tommy_size_t key_len); /** - * String hash function with a 32-bit result. - * Modified implementation of a5hash32 by Aleksey Vaneev, - * from https://github.com/avaneev/a5hash + * String hash function with a 32 bits result. + * Implementation is based on Robert Jenkins "lookup3" hash 32 bits version, + * from http://www.burtleburtle.net/bob/hash/doobs.html, function hashlittle(). * - * This hash is designed to handle strings with an unknown length. - * It is implemented inline using strlen() followed by ::tommy_hash_u32. - * This gives better performance than scanning for the terminator while hashing. - * It uses the same modified seed mixing and differs from upstream a5hash32. + * This hash is designed to handle strings with an unknown length. If you + * know the string length, the other hash functions are surely faster. * * \param init_val Initialization value. * Using a different initialization value, you can generate a completely different set of hash values. * Use 0 if not relevant. - * \param void_key Pointer to the string to hash. It has to be 0-terminated. - * \return The 32-bit hash value. + * \param void_key Pointer to the string to hash. It has to be 0 terminated. * \note - * This function is endianness independent and alignment independent. - * It produces identical hash results to ::tommy_hash_u32 for 0-terminated strings. - * Compatibility across different versions of the library is not guaranteed: - * this hash is intended for in-memory runtime use only and not for persistent storage. - * It is non-cryptographic and is not designed to protect against malicious collision attacks. + * This function is endianness independent. + * \return The hash value of 32 bits. */ -tommy_inline tommy_uint32_t tommy_strhash_u32(tommy_uint32_t init_val, const void* void_key) -{ - return tommy_hash_u32(init_val, void_key, strlen(tommy_cast(const char*, void_key))); -} +TOMMY_API tommy_uint32_t tommy_strhash_u32(tommy_uint32_t init_val, const void* void_key); /** - * String hash function with a 64-bit result. - * Implementation of a5hash (64-bit) by Aleksey Vaneev, - * from https://github.com/avaneev/a5hash - * - * This hash is designed to handle strings with an unknown length. - * It is implemented inline using strlen() followed by ::tommy_hash_u64. - * This gives better performance than scanning for the terminator while hashing. - * - * \param init_val Initialization value. - * Using a different initialization value, you can generate a completely different set of hash values. - * Use 0 if not relevant. - * \param void_key Pointer to the string to hash. It has to be 0-terminated. - * \return The 64-bit hash value. - * \note - * This function is endianness independent and alignment independent. - * It produces identical hash results to ::tommy_hash_u64 for 0-terminated strings. - * Compatibility across different versions of the library is not guaranteed: - * this hash is intended for in-memory runtime use only and not for persistent storage. - * It is non-cryptographic and is not designed to protect against malicious collision attacks. - */ -tommy_inline tommy_uint64_t tommy_strhash_u64(tommy_uint64_t init_val, const void* void_key) -{ - return tommy_hash_u64(init_val, void_key, strlen(tommy_cast(const char*, void_key))); -} - -/** - * Reversible 32-bit integer hash function. + * Integer reversible hash function for 32 bits. * Implementation of the Robert Jenkins "4-byte Integer Hashing", * from http://burtleburtle.net/bob/hash/integer.html - * \param key The 32-bit integer to hash. - * \return The 32-bit hash value. */ tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) { @@ -156,11 +90,9 @@ tommy_inline tommy_uint32_t tommy_inthash_u32(tommy_uint32_t key) } /** - * Reversible 64-bit integer hash function. + * Integer reversible hash function for 64 bits. * Implementation of the Thomas Wang "Integer Hash Function", * from http://web.archive.org/web/20071223173210/http://www.concentric.net/~Ttwang/tech/inthash.htm - * \param key The 64-bit integer to hash. - * \return The 64-bit hash value. */ tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) { @@ -175,146 +107,4 @@ tommy_inline tommy_uint64_t tommy_inthash_u64(tommy_uint64_t key) return key; } -/** - * Reversible 32-bit integer hash function using multiplications. - * Implementation of the MurmurHash3 fmix32 finalizer by Austin Appleby, - * from https://github.com/aappleby/smhasher/blob/master/src/MurmurHash3.cpp - * \param key The 32-bit integer to hash. - * \return The 32-bit hash value. - * \note Requires an architecture with fast 32-bit integer multiplication - * for good performance. See tommy_inthash_u32() for an alternative. - */ -tommy_inline tommy_uint32_t tommy_mulhash_u32(tommy_uint32_t key) -{ - key ^= key >> 16; - key *= 0x85ebca6bU; - key ^= key >> 13; - key *= 0xc2b2ae35U; - key ^= key >> 16; - - return key; -} - -/** - * Reversible 64-bit integer hash function using multiplications. - * Implementation of the SplitMix64 finalizer, - * from https://prng.di.unimi.it/splitmix64.c - * \param key The 64-bit integer to hash. - * \return The 64-bit hash value. - * \note Requires an architecture with fast 64-bit integer multiplication - * for good performance. It is generally not recommended on 32-bit architectures, - * where these multiplications are expensive. Consider tommy_inthash_u64() instead. - */ -tommy_inline tommy_uint64_t tommy_mulhash_u64(tommy_uint64_t key) -{ - key ^= key >> 30; - key *= 0xbf58476d1ce4e5b9ULL; - key ^= key >> 27; - key *= 0x94d049bb133111ebULL; - key ^= key >> 31; - - return key; -} - -/******************************************************************************/ -/* multiplication */ - -/** - * Multiplies two 64-bit unsigned numbers to obtain the full 128-bit product. - * - * \param A Address of the first operand, overwritten with the low 64 bits of the product. - * \param B Address of the second operand, overwritten with the high 64 bits of the product. - */ -#if defined(_MSC_VER) && defined(_M_X64) -/* - * MSVC x64 does not support inline assembly, so use its full-product intrinsic. - */ -#pragma intrinsic(_umul128) -tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) -{ - tommy_uint64_t high; - tommy_uint64_t low = _umul128(*A, *B, &high); - *A = low; - *B = high; -} -#elif defined(_MSC_VER) && defined(_M_ARM64) -#pragma intrinsic(__umulh) -tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) -{ - tommy_uint64_t a = *A; - tommy_uint64_t b = *B; - *A = a * b; - *B = __umulh(a, b); -} -#elif defined(__x86_64__) && (defined(__GNUC__) || defined(__clang__)) -/** - * Keep this before __SIZEOF_INT128__ because the explicit mulq generates more efficient code. - */ -tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) -{ - tommy_uint64_t a = *A; - tommy_uint64_t b = *B; - tommy_uint64_t low, high; - - /* require a register to avoid Clang spilling the operand before mulq */ - __asm__ ( - "mulq %3" - : "=a" (low), "=d" (high) - : "a" (a), "r" (b) - : "cc" - ); - - *A = low; - *B = high; -} -#elif defined(__SIZEOF_INT128__) -tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) -{ - __uint128_t product = (__uint128_t)*A * *B; - *A = product; - *B = product >> 64; -} -#elif defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__)) -/* - * Keep this after __SIZEOF_INT128__ because the assembly path failed optimized - * hash tests on Apple clang 21.0.0. See https://github.com/amadvance/snapraid/issues/43. - */ -tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) -{ - tommy_uint64_t a = *A; - tommy_uint64_t b = *B; - tommy_uint64_t low = a * b; - tommy_uint64_t high; - __asm__ ( - "umulh %0, %1, %2" - : "=r" (high) - : "r" (a), "r" (b) - ); - *A = low; - *B = high; -} -#else -tommy_inline void tommy_mum(tommy_uint64_t* A, tommy_uint64_t* B) -{ - tommy_uint64_t a = *A; - tommy_uint64_t b = *B; - tommy_uint64_t a_lo = (tommy_uint32_t)a; - tommy_uint64_t a_hi = a >> 32; - tommy_uint64_t b_lo = (tommy_uint32_t)b; - tommy_uint64_t b_hi = b >> 32; - tommy_uint64_t p0 = a_lo * b_lo; - tommy_uint64_t p1 = a_lo * b_hi; - tommy_uint64_t p2 = a_hi * b_lo; - tommy_uint64_t p3 = a_hi * b_hi; - - /* retain the carry from the middle words in the high half */ - tommy_uint64_t middle = p1 + (p0 >> 32) + (tommy_uint32_t)p2; - tommy_uint64_t low = (middle << 32) | (tommy_uint32_t)p0; - tommy_uint64_t high = p3 + (middle >> 32) + (p2 >> 32); - *A = low; - *B = high; -} #endif - -#endif - diff --git a/tommyds/tommyhashdyn.c b/tommyds/tommyhashdyn.c index 6775d5dd..ce2587fb 100644 --- a/tommyds/tommyhashdyn.c +++ b/tommyds/tommyhashdyn.c @@ -4,8 +4,6 @@ #include "tommyhashdyn.h" #include "tommylist.h" -#include /* for memset */ - /******************************************************************************/ /* hashdyn */ @@ -25,92 +23,61 @@ TOMMY_API void tommy_hashdyn_done(tommy_hashdyn* hashdyn) tommy_free(hashdyn->bucket); } -TOMMY_API void tommy_hashdyn_clear(tommy_hashdyn* hashdyn) -{ - memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); - hashdyn->count = 0; -} - /** * Resize the bucket vector. */ -static void tommy_imp_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket_bit) +static void tommy_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bucket_bit) { - tommy_size_t bucket_bit = hashdyn->bucket_bit; - tommy_size_t bucket_max = hashdyn->bucket_max; - tommy_size_t new_bucket_max = (tommy_size_t)1 << new_bucket_bit; - tommy_size_t new_bucket_mask = new_bucket_max - 1; + tommy_size_t bucket_bit; + tommy_size_t bucket_max; + tommy_size_t new_bucket_max; + tommy_size_t new_bucket_mask; tommy_hashdyn_node** new_bucket; - if (!hashdyn->count) { - /* empty table: allocate zeroed memory with calloc without looping */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); - } else if (new_bucket_bit > bucket_bit) { - if (new_bucket_bit == bucket_bit + 1) { - /* grow by 1 bit: zero the two buckets inline in loop to preserve cache locality */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); + bucket_bit = hashdyn->bucket_bit; + bucket_max = hashdyn->bucket_max; - for (tommy_size_t i = 0; i < bucket_max; ++i) { - new_bucket[i] = 0; - new_bucket[i + bucket_max] = 0; + new_bucket_max = (tommy_size_t)1 << new_bucket_bit; + new_bucket_mask = new_bucket_max - 1; - tommy_hashdyn_node* j = hashdyn->bucket[i]; - if (j && !j->next) { - /* growth keeps old buckets separate, and singleton links are already valid */ - new_bucket[j->index & new_bucket_mask] = j; - continue; - } - while (j) { - tommy_hashdyn_node* j_next = j->next; - tommy_size_t pos = j->index & new_bucket_mask; - if (new_bucket[pos]) - tommy_list_insert_tail_not_empty(new_bucket[pos], j); - else - tommy_list_insert_first(&new_bucket[pos], j); - j = j_next; - } - } - } else { - /* grow by multiple bits with elements: allocate zeroed memory and reinsert */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_calloc(new_bucket_max, sizeof(tommy_hashdyn_node*))); + /* allocate the new vector using malloc() and not calloc() */ + /* because data is fully initialized in the update process */ + new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); - for (tommy_size_t i = 0; i < bucket_max; ++i) { - tommy_hashdyn_node* j = hashdyn->bucket[i]; + /* reinsert all the elements */ + if (new_bucket_bit > bucket_bit) { + tommy_size_t i; - if (j && !j->next) { - /* growth keeps old buckets separate, and singleton links are already valid */ - new_bucket[j->index & new_bucket_mask] = j; - continue; - } - while (j) { - tommy_hashdyn_node* j_next = j->next; - tommy_size_t pos = j->index & new_bucket_mask; - if (new_bucket[pos]) - tommy_list_insert_tail_not_empty(new_bucket[pos], j); - else - tommy_list_insert_first(&new_bucket[pos], j); - j = j_next; - } + /* grow */ + for (i = 0; i < bucket_max; ++i) { + tommy_hashdyn_node* j; + + /* setup the new two buckets */ + new_bucket[i] = 0; + new_bucket[i + bucket_max] = 0; + + /* reinsert the bucket */ + j = hashdyn->bucket[i]; + while (j) { + tommy_hashdyn_node* j_next = j->next; + tommy_size_t pos = j->index & new_bucket_mask; + if (new_bucket[pos]) + tommy_list_insert_tail_not_empty(new_bucket[pos], j); + else + tommy_list_insert_first(&new_bucket[pos], j); + j = j_next; } } } else { - /* all buckets are overwritten, no pre-zeroing needed */ - new_bucket = tommy_cast(tommy_hashdyn_node**, tommy_malloc(new_bucket_max * sizeof(tommy_hashdyn_node*))); + tommy_size_t i; - if (new_bucket_bit + 1 == bucket_bit) { - /* shrink by 1 bit: each new bucket joins exactly two old buckets */ - for (tommy_size_t i = 0; i < new_bucket_max; ++i) { - new_bucket[i] = hashdyn->bucket[i]; - tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); - } - } else { - /* shrink by multiple bits */ - for (tommy_size_t i = 0; i < new_bucket_max; ++i) { - /* all old buckets with the same new modulus must be concatenated. */ - new_bucket[i] = hashdyn->bucket[i]; - for (tommy_size_t j = i + new_bucket_max; j < bucket_max; j += new_bucket_max) - tommy_list_concat(&new_bucket[i], &hashdyn->bucket[j]); - } + /* shrink */ + for (i = 0; i < new_bucket_max; ++i) { + /* setup the new bucket with the lower bucket*/ + new_bucket[i] = hashdyn->bucket[i]; + + /* concat the upper bucket */ + tommy_list_concat(&new_bucket[i], &hashdyn->bucket[i + new_bucket_max]); } } @@ -126,47 +93,21 @@ static void tommy_imp_hashdyn_resize(tommy_hashdyn* hashdyn, tommy_uint_t new_bu /** * Grow. */ -tommy_inline void tommy_imp_hashdyn_grow_step(tommy_hashdyn* hashdyn) +tommy_inline void hashdyn_grow_step(tommy_hashdyn* hashdyn) { - /* grow if at least 50% full */ + /* grow if more than 50% full */ if (hashdyn->count >= hashdyn->bucket_max / 2) - tommy_imp_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); + tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit + 1); } /** * Shrink. */ -tommy_inline void tommy_imp_hashdyn_shrink_step(tommy_hashdyn* hashdyn) +tommy_inline void hashdyn_shrink_step(tommy_hashdyn* hashdyn) { - /* shrink if at most 12.5% full */ + /* shrink if less than 12.5% full */ if (hashdyn->count <= hashdyn->bucket_max / 8 && hashdyn->bucket_bit > TOMMY_HASHDYN_BIT) - tommy_imp_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); -} - -TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count) -{ - tommy_uint_t bucket_bit = tommy_ilog2(count | 1) + 2; - - if (bucket_bit < TOMMY_HASHDYN_BIT) - bucket_bit = TOMMY_HASHDYN_BIT; - - if (bucket_bit <= hashdyn->bucket_bit) - return; - - tommy_imp_hashdyn_resize(hashdyn, bucket_bit); -} - -TOMMY_API void tommy_hashdyn_shrink(tommy_hashdyn* hashdyn) -{ - tommy_uint_t bucket_bit = tommy_ilog2(hashdyn->count | 1) + 2; - - if (bucket_bit < TOMMY_HASHDYN_BIT) - bucket_bit = TOMMY_HASHDYN_BIT; - - if (bucket_bit >= hashdyn->bucket_bit) - return; - - tommy_imp_hashdyn_resize(hashdyn, bucket_bit); + tommy_hashdyn_resize(hashdyn, hashdyn->bucket_bit - 1); } TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash) @@ -179,42 +120,7 @@ TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* ++hashdyn->count; - tommy_imp_hashdyn_grow_step(hashdyn); -} - -TOMMY_API void* tommy_hashdyn_insert_unique(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) -{ - tommy_hashdyn_node** bucket = &hashdyn->bucket[hash & hashdyn->bucket_mask]; - tommy_hashdyn_node* i = *bucket; - - while (i) { - /* compare keys only for matching full hashes, as the bucket may contain different hashes */ - if (i->index == hash && cmp(cmp_arg, i->data) == 0) - return i->data; - i = i->next; - } - - /* insert before resizing, while the bucket selected for the search is still valid */ - tommy_list_insert_tail(bucket, node, data); - - node->index = hash; - - ++hashdyn->count; - - tommy_imp_hashdyn_grow_step(hashdyn); - - return data; -} - -TOMMY_API void tommy_hashdyn_rehash_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, tommy_hash_t hash) -{ - if (node->index == hash) - return; - - /* unlink using the stored hash, without invoking the resize policy. */ - tommy_list_remove_existing(&hashdyn->bucket[node->index & hashdyn->bucket_mask], node); - node->index = hash; - tommy_list_insert_tail(&hashdyn->bucket[hash & hashdyn->bucket_mask], node, node->data); + hashdyn_grow_step(hashdyn); } TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) @@ -225,7 +131,7 @@ TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hash --hashdyn->count; - tommy_imp_hashdyn_shrink_step(hashdyn); + hashdyn_shrink_step(hashdyn); return node->data; } @@ -242,7 +148,7 @@ TOMMY_API void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* --hashdyn->count; - tommy_imp_hashdyn_shrink_step(hashdyn); + hashdyn_shrink_step(hashdyn); return node->data; } @@ -256,8 +162,9 @@ TOMMY_API void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* { tommy_size_t bucket_max = hashdyn->bucket_max; tommy_hashdyn_node** bucket = hashdyn->bucket; + tommy_size_t pos; - for (tommy_size_t pos = 0; pos < bucket_max; ++pos) { + for (pos = 0; pos < bucket_max; ++pos) { tommy_hashdyn_node* node = bucket[pos]; while (node) { @@ -272,8 +179,9 @@ TOMMY_API void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_a { tommy_size_t bucket_max = hashdyn->bucket_max; tommy_hashdyn_node** bucket = hashdyn->bucket; + tommy_size_t pos; - for (tommy_size_t pos = 0; pos < bucket_max; ++pos) { + for (pos = 0; pos < bucket_max; ++pos) { tommy_hashdyn_node* node = bucket[pos]; while (node) { @@ -284,180 +192,67 @@ TOMMY_API void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_a } } -TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(const tommy_hashdyn* hashdyn) +TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn) { return hashdyn->bucket_max * (tommy_size_t)sizeof(hashdyn->bucket[0]) + tommy_hashdyn_count(hashdyn) * (tommy_size_t)sizeof(tommy_hashdyn_node); } +/** + * \brief Transfers all elements from the dynamic hashtable into a tommy_list. + * + * Removes every element from the \p hashdyn hashtable and inserts them + * into the provided \p list (at the tail), preserving the per-bucket order + * but not guaranteeing any particular global order. + * + * After the call: + * - the hashtable is left completely empty (\c count = 0, all buckets = NULL) + * - the target list contains all the elements that were previously in the hashtable + * + * This function is useful when you need to: + * - extract all elements to process/sort them outside the hash table + * - convert the hashtable into a list for sequential iteration + * - prepare for a full clear + re-insertion with different hash/ordering + * - move ownership of the nodes to a list-based container + * + * \note The operation is O(n) where n is the number of elements. + * \note No memory allocation is performed. + * \note The relative order of elements that were in the same bucket is preserved, + * but the order among different buckets is bucket-order dependent. + * + * Typical usage pattern: + * \code + * tommy_list all_elements; + * tommy_list_init(&all_elements); + * + * // move everything out of the hashtable into the list + * tommy_hashdyn_to_list(&hashdyn, &all_elements); + * + * // now you can sort, filter, process sequentially, etc. + * tommy_list_sort(&all_elements, compare_by_value); + * \endcode + * + * \param hashdyn The dynamic hashtable to drain + * \param list The list that will receive all the elements + */ TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list) { - tommy_builder builder; - tommy_node* builder_tail = tommy_builder_init(&builder); + tommy_size_t bucket_max = hashdyn->bucket_max; + tommy_hashdyn_node** bucket = hashdyn->bucket; + tommy_size_t pos; - /* clear buckets as they are transferred to avoid a second scan */ - for (tommy_size_t pos = 0; pos < hashdyn->bucket_max; ++pos) { - tommy_node* head = hashdyn->bucket[pos]; - if (head) { - tommy_node* tail = head->prev; - builder_tail = tommy_builder_concat(builder_tail, head, tail); + /* move everything to the list */ + for (pos = 0; pos < bucket_max; ++pos) { + tommy_hashdyn_node* node = bucket[pos]; + + while (node) { + tommy_node* node_next = node->next; + tommy_list_insert_tail(list, node, node->data); + node = node_next; } - hashdyn->bucket[pos] = 0; } - tommy_list_concat_builder(list, &builder, builder_tail); + /* clear all */ + memset(hashdyn->bucket, 0, hashdyn->bucket_max * sizeof(tommy_hashdyn_node*)); hashdyn->count = 0; } - -/** - * Compares hashes in least-significant-bit-first order, with 0 before 1. - * - * The first differing bit decides the order: first precedes second when - * second has a 1 there. Equal hashes compare false. - */ -tommy_inline tommy_bool_t tommy_imp_hashdyn_before(tommy_hash_t first, tommy_hash_t second) -{ - /* mark the positions where the hashes differ using XOR */ - tommy_hash_t diff = first ^ second; - - /* isolate the lowest differing bit using two's complement */ - tommy_hash_t bit = diff & -diff; - - /* a 1 in second means first has 0 there; equal hashes yield bit == 0 */ - return (second & bit) != 0; -} - -/** - * Gets the adjacent active bucket in least-significant-bit-first order. - * Returns bucket_max when there is no adjacent bucket. - * - * Traverses the virtual binary tree of buckets in bit-reversed order. - * - * For example, with 8 buckets the forward traversal order is: - * 0, 4, 2, 6, 1, 5, 3, 7 - * - * When resizing to 16 buckets: - * 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 - */ -tommy_inline tommy_size_t tommy_imp_hashdyn_step(tommy_hashdyn* hashdyn, tommy_size_t pos, tommy_bool_t reverse) -{ - tommy_size_t bit = (hashdyn->bucket_mask + 1) / 2; - - /* propagate binary carry in bit-reversed order from MSB downwards */ - while (bit && ((pos & bit) != 0) != reverse) { - pos ^= bit; - bit >>= 1; - } - - /* carry wrapped around the root of the tree: end of traversal */ - if (!bit) - return hashdyn->bucket_max; - pos ^= bit; - - return pos; -} - -/** - * Finds the first or last node starting at an active bucket. - * - * Scans buckets in least-significant-bit-first order until a non-empty bucket - * is found, then returns its extreme element (minimum for forward, maximum for reverse). - */ -tommy_inline tommy_hashdyn_node* tommy_imp_hashdyn_limit(tommy_hashdyn* hashdyn, tommy_size_t pos, tommy_bool_t reverse) -{ - while (pos != hashdyn->bucket_max) { - tommy_hashdyn_node* best = 0; - for (tommy_hashdyn_node* i = hashdyn->bucket[pos]; i; i = i->next) { - /* equal hashes retain insertion order, so the reverse limit selects the last */ - if (reverse) { - if (!best || !tommy_imp_hashdyn_before(i->index, best->index)) - best = i; - } else { - if (!best || tommy_imp_hashdyn_before(i->index, best->index)) - best = i; - } - } - if (best) - return best; - pos = tommy_imp_hashdyn_step(hashdyn, pos, reverse); - } - return 0; -} - -/** - * Finds a neighbor using the current bucket layout and a resize-independent order. - * - * First searches for the closest neighbor within the current bucket. - * If none exists, advances to the next non-empty active bucket in LSB order. - */ -tommy_inline tommy_hashdyn_node* tommy_imp_hashdyn_neighbor(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, tommy_bool_t reverse) -{ - tommy_size_t pos = node->index & hashdyn->bucket_mask; - tommy_hashdyn_node* best = 0; - tommy_bool_t passed = 0; - - /* - * Bucket links can change during grow and shrink. Compare full hashes instead - * of following those links as iteration order, so no remaining node moves - * behind the iterator. Equal hashes always stay in one bucket in insertion order. - */ - for (tommy_hashdyn_node* i = hashdyn->bucket[pos]; i; i = i->next) { - if (i == node) { - passed = 1; - } else if (i->index == node->index) { - /* for identical hashes, follow insertion order relative to the current node */ - if (reverse && !passed) - best = i; - else if (!reverse && passed) - return i; - } else if (reverse) { - if (tommy_imp_hashdyn_before(i->index, node->index)) { - /* find the closest neighbor within the bucket in LSB order */ - if (!best || !tommy_imp_hashdyn_before(i->index, best->index)) - best = i; - } - } else { - if (tommy_imp_hashdyn_before(node->index, i->index)) { - /* find the closest neighbor within the bucket in LSB order */ - if (!best || tommy_imp_hashdyn_before(i->index, best->index)) - best = i; - } - } - } - if (best) - return best; - - /* bucket exhausted: advance to the next non-empty bucket in LSB order */ - return tommy_imp_hashdyn_limit(hashdyn, tommy_imp_hashdyn_step(hashdyn, pos, reverse), reverse); -} - -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_head(tommy_hashdyn* hashdyn) -{ - /* start forward traversal at bucket 0, which has all LSB bits at zero */ - return tommy_imp_hashdyn_limit(hashdyn, 0, 0); -} - -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_tail(tommy_hashdyn* hashdyn) -{ - /* start reverse traversal at the all-ones bucket, which has the highest LSB value */ - return tommy_imp_hashdyn_limit(hashdyn, hashdyn->bucket_mask, 1); -} - -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_next(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) -{ - /* adjacent equal hashes are consecutive in insertion order */ - if (node->next && node->next->index == node->index) - return node->next; - - return tommy_imp_hashdyn_neighbor(hashdyn, node, 0); -} - -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_prev(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node) -{ - /* equal hashes follow insertion order, but the bucket head's prev points to the tail */ - if (node != hashdyn->bucket[node->index & hashdyn->bucket_mask] && node->prev->index == node->index) - return node->prev; - - return tommy_imp_hashdyn_neighbor(hashdyn, node, 1); -} - diff --git a/tommyds/tommyhashdyn.h b/tommyds/tommyhashdyn.h index 91018fe0..5ed7ed7a 100644 --- a/tommyds/tommyhashdyn.h +++ b/tommyds/tommyhashdyn.h @@ -4,18 +4,18 @@ /** \file * Dynamic chained hashtable. * - * This hashtable resizes dynamically. It starts with the minimal size of 16 buckets. - * It doubles the size when it reaches a load factor greater than or equal to 0.5 - * and it halves the size when the load factor is less than or equal to 0.125. + * This hashtable resizes dynamically. It starts with the minimal size of 16 buckets, it doubles + * the size when it reaches a load factor greater than 0.5 and it halves the size when the load + * factor is lower than 0.125. * * All the elements are reallocated in a single resize operation done inside * tommy_hashdyn_insert() or tommy_hashdyn_remove(). * - * Note that the resize operation takes approximately 100 [ms] with 1 million elements, - * and 1 [second] with 10 million elements. This could be a problem in real-time applications. + * Note that the resize operation takes approximately 100 [ms] with 1 million of elements, + * and 1 [second] with 10 millions. This could be a problem in real-time applications. * - * The resize also fragments the heap, as it involves allocating a double-sized table, copying elements, - * and deallocating the older table, leaving a big hole in the heap. + * The resize also **fragments** the heap, as it involves allocating a double-sized table, **copying** elements, + * and deallocating the older table, **leaving** a big hole in the heap. * * The ::tommy_hashlin hashtable fixes both problems. * @@ -30,36 +30,36 @@ * To insert elements in the hashtable you have to call tommy_hashdyn_insert() for * each element. * In the insertion call you have to specify the address of the node, the - * address of the element, and the hash value of the key to use. - * The address of the element is used to initialize the tommy_node::data field - * of the node, and the hash to initialize the tommy_node::index field. + * address of the object, and the hash value of the key to use. + * The address of the object is used to initialize the tommy_node::data field + * of the node, and the hash to initialize the tommy_node::key field. * * \code - * struct element { + * struct object { * int value; * // other fields * tommy_node node; * }; * - * struct element* elem = malloc(sizeof(struct element)); // creates the element + * struct object* obj = malloc(sizeof(struct object)); // creates the object * - * elem->value = ...; // initializes the element + * obj->value = ...; // initializes the object * - * tommy_hashdyn_insert(&hashdyn, &elem->node, elem, tommy_inthash_u32(elem->value)); // inserts the element + * tommy_hashdyn_insert(&hashdyn, &obj->node, obj, tommy_inthash_u32(obj->value)); // inserts the object * \endcode * - * To find an element in the hashtable you have to call tommy_hashdyn_search() + * To find an element in the hashtable you have to call tommy_hashtable_search() * providing a comparison function, its argument, and the hash of the key to search. * * \code - * int compare(const void* arg, const void* elem) + * int compare(const void* arg, const void* obj) * { - * return *(const int*)arg != ((const struct element*)elem)->value; + * return *(const int*)arg != ((const struct object*)obj)->value; * } * * int value_to_find = 1; - * struct element* elem = tommy_hashdyn_search(&hashdyn, compare, &value_to_find, tommy_inthash_u32(value_to_find)); - * if (!elem) { + * struct object* obj = tommy_hashdyn_search(&hashdyn, compare, &value_to_find, tommy_inthash_u32(value_to_find)); + * if (!obj) { * // not found * } else { * // found @@ -67,7 +67,7 @@ * \endcode * * To iterate over all the elements in the hashtable with the same key, you have to - * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until 0. + * use tommy_hashdyn_bucket() and follow the tommy_node::next pointer until NULL. * You have also to check explicitly for the key, as the bucket may contain * different keys. * @@ -75,10 +75,10 @@ * int value_to_find = 1; * tommy_node* i = tommy_hashdyn_bucket(&hashdyn, tommy_inthash_u32(value_to_find)); * while (i) { - * struct element* elem = i->data; // gets the element pointer + * struct object* obj = i->data; // gets the object pointer * - * if (elem->value == value_to_find) { - * printf("%d\n", elem->value); // process the element + * if (obj->value == value_to_find) { + * printf("%d\n", obj->value); // process the object * } * * i = i->next; // goes to the next element @@ -90,9 +90,9 @@ * and remove. * * \code - * struct element* elem = tommy_hashdyn_remove(&hashdyn, compare, &value_to_remove, tommy_inthash_u32(value_to_remove)); - * if (elem) { - * free(elem); // frees the element allocated memory + * struct object* obj = tommy_hashdyn_remove(&hashdyn, compare, &value_to_remove, tommy_inthash_u32(value_to_remove)); + * if (obj) { + * free(obj); // frees the object allocated memory * } * \endcode * @@ -105,60 +105,15 @@ * * If you need to iterate over all the elements in the hashtable, you can use * tommy_hashdyn_foreach() or tommy_hashdyn_foreach_arg(). - * For traversal that permits insertions and removals, use tommy_hashdyn_head() - * and tommy_hashdyn_next(), or tommy_hashdyn_tail() and tommy_hashdyn_prev(). - * Hashes are compared from the least significant bit to the most significant - * bit, with 0 before 1. Equal hashes are visited in insertion order. - * This order is independent of the number of buckets and progressive resizing. - * It differs from the bucket order used by foreach and to_list. - * - * \anchor tommy_hashdyn_iterator_validity - * Iterators are pointers to nodes currently contained in the hashtable. - * Inserting elements or removing other nodes does not invalidate them or - * change the relative iteration order of the remaining elements. - * A forward or backward traversal visits every original element that remains - * contained, even if removals cause the table to shrink or insertions grow it. - * New elements can be visited if inserted after the current position in the - * traversal direction; elements inserted before it need not be visited. - * - * Removing the pointed node invalidates its use as an iterator, even if its - * storage has not been freed. Obtain the next or previous node before removal, - * and keep that saved node contained until continuing the traversal. - * - * \code - * tommy_hashdyn_node* i = tommy_hashdyn_head(&hashdyn); - * while (i) { - * tommy_hashdyn_node* next = tommy_hashdyn_next(&hashdyn, i); - * if (should_remove(i->data)) { - * free(tommy_hashdyn_remove_existing(&hashdyn, i)); - * } - * i = next; - * } - * \endcode - * - * Changing a hash with tommy_hashdyn_rehash_existing() can change iteration - * order, so the traversal guarantee does not cover rehashing during traversal. - * After swapping containers, use the container that now contains the node. - * Clearing or deinitializing the container invalidates all its iterators. - * Concurrent traversal and modification require external synchronization. - * - * Iterator calls scan bucket contents without allocating or modifying memory. - * With a good hash function and normal dynamic resizing, a full traversal - * runs in expected O(n) time. Under the same hash distribution assumption, - * sparse tables require expected O(n + b) time, where n is the number of - * elements and b is the number of buckets. Such tables can arise after - * tommy_hashdyn_reserve(), tommy_hashdyn_clear(), or tommy_hashdyn_to_list(). - * In pathological cases with severe hash collisions, traversal can take - * O(n^2 + b) time. - * Use foreach for guaranteed O(n + b) traversal when no insertions or removals - * are needed. + * If you need a more precise control with a real iteration, you have to insert + * all the elements also in a ::tommy_list, and use the list to iterate. + * See the \ref multiindex example for more detail. */ -#ifndef TOMMY_HASHDYN_H -#define TOMMY_HASHDYN_H +#ifndef __TOMMYHASHDYN_H +#define __TOMMYHASHDYN_H #include "tommyhash.h" -#include "tommylist.h" /******************************************************************************/ /* hashdyn */ @@ -171,7 +126,7 @@ /** * Hashtable node. - * This is the node that you have to include inside your elements. + * This is the node that you have to include inside your objects. */ typedef tommy_node tommy_hashdyn_node; @@ -189,7 +144,6 @@ typedef struct tommy_hashdyn_struct { /** * Initializes the hashtable. - * \param hashdyn The hashtable. */ TOMMY_API void tommy_hashdyn_init(tommy_hashdyn* hashdyn); @@ -198,99 +152,21 @@ TOMMY_API void tommy_hashdyn_init(tommy_hashdyn* hashdyn); * * You can call this function with elements still contained, * but such elements are not going to be freed by this call. - * \param hashdyn The hashtable. */ TOMMY_API void tommy_hashdyn_done(tommy_hashdyn* hashdyn); -/** - * Exchanges the complete state of two initialized hashtables, including bucket allocations. - * The hashtables must not share nodes. Nodes and elements are not accessed or modified. - * Existing node pointers remain valid and belong to the other hashtable. - * Passing the same hashtable twice has no effect. Both hashtables remain usable and deinitializable. - * \param first The first hashtable. - * \param second The second hashtable. - * \note This operation is O(1) with respect to the number of elements. - */ -tommy_inline void tommy_hashdyn_swap(tommy_hashdyn* first, tommy_hashdyn* second) -{ - tommy_hashdyn tmp = *first; - *first = *second; - *second = tmp; -} - -/** - * Removes all elements, preserving the allocated buckets. - * Call tommy_hashdyn_shrink() after clearing to restore the initial capacity. - * The hashtable remains initialized and can be reused immediately. - * Elements are not freed and nodes are not accessed or modified. - * Their links must not be used to traverse the previous contents. - * You can call this function after tommy_hashdyn_foreach() has freed the elements. - * Subsequent insertions and removals retain the normal resizing policy. - * \param hashdyn The hashtable. - * \note This operation is O(b), where b is the number of buckets. - */ -TOMMY_API void tommy_hashdyn_clear(tommy_hashdyn* hashdyn); - -/** - * Pre-allocates buckets for the specified number of elements. - * - * It ensures that the hashtable has enough buckets allocated to contain - * the specified number of elements without triggering a dynamic resize on insertion. - * Subsequent removals retain the normal shrinking policy and may reduce - * the allocated capacity. - * \param hashdyn The hashtable. - * \param count Number of elements to reserve space for. - */ -TOMMY_API void tommy_hashdyn_reserve(tommy_hashdyn* hashdyn, tommy_size_t count); - -/** - * Shrinks the allocated buckets to fit the current number of elements. - * The target is the smallest power of 2, at least 16, for which - * the number of elements is strictly less than half the number of buckets. - * If the current allocation is already no larger than the target, nothing is done. - * An empty hashtable is reduced to 16 buckets. - * Elements are not freed, and the tommy_node::data and tommy_node::index fields - * are left unchanged. The order of elements with the same hash is preserved. - * \param hashdyn The hashtable. - * \note This operation is O(b), where b is the previous number of buckets. - */ -TOMMY_API void tommy_hashdyn_shrink(tommy_hashdyn* hashdyn); - /** * Inserts an element in the hashtable. - * \param hashdyn The hashtable. - * \param node Pointer to the node embedded into the element to insert. - * \param data Pointer to the element to insert. - * \param hash Hash of the element to insert. */ TOMMY_API void tommy_hashdyn_insert(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_hash_t hash); -/** - * Inserts an element only if no equal element is already contained. - * If found, the first equal element is returned, - * and the hashtable and candidate node are left unchanged. - * Otherwise, the candidate is inserted using the normal insertion policy, - * and data is returned. - * Elements are not freed by this call. - * \param hashdyn The hashtable. - * \param node The candidate node. It must not belong to any container. - * \param data Pointer to the element to insert. - * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. - * The function should return 0 for equal elements, non-zero for different elements. - * \param cmp_arg Compare argument describing the candidate key. - * \param hash Hash of the candidate key, consistent with the comparison function. - * \return The first equal element, or data if the candidate was inserted. - */ -TOMMY_API void* tommy_hashdyn_insert_unique(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, void* data, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash); - /** * Searches and removes an element from the hashtable. * You have to provide a compare function and the hash of the element you want to remove. * If the element is not found, 0 is returned. * If more equal elements are present, the first one is removed. - * \param hashdyn The hashtable. * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. - * The function should return 0 for equal elements, non-zero for different elements. + * The function should return 0 for equal elements, anything other for different elements. * \param cmp_arg Compare argument passed as first argument of the compare function. * \param hash Hash of the element to find and remove. * \return The removed element, or 0 if not found. @@ -300,9 +176,8 @@ TOMMY_API void* tommy_hashdyn_remove(tommy_hashdyn* hashdyn, tommy_search_func* /** * Gets the bucket of the specified hash. * The bucket is guaranteed to contain ALL the elements with the specified hash, - * but it can also contain others. - * You can access elements in the bucket following the tommy_node::next pointer until 0. - * \param hashdyn The hashtable. + * but it can contain also others. + * You can access elements in the bucket following the ::next pointer until 0. * \param hash Hash of the element to find. * \return The head of the bucket, or 0 if empty. */ @@ -311,124 +186,36 @@ tommy_inline tommy_hashdyn_node* tommy_hashdyn_bucket(tommy_hashdyn* hashdyn, to return hashdyn->bucket[hash & hashdyn->bucket_mask]; } -/** - * Finds a node in the hashtable. - * You have to provide a compare function and the hash of the element you want to find. - * If more equal elements are present, the first one is returned. - * \param hashdyn The hashtable. - * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. - * The function should return 0 for equal elements, non-zero for different elements. - * \param cmp_arg Compare argument passed as first argument of the compare function. - * \param hash Hash of the element to find. - * \return The first node found, or 0 if none. - */ -tommy_inline tommy_hashdyn_node* tommy_hashdyn_find(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) -{ - tommy_hashdyn_node* i = tommy_hashdyn_bucket(hashdyn, hash); - - while (i) { - /* we first check if the hash matches, as in the same bucket we may have multiple hash values */ - if (i->index == hash && cmp(cmp_arg, i->data) == 0) - return i; - i = i->next; - } - return 0; -} - /** * Searches an element in the hashtable. * You have to provide a compare function and the hash of the element you want to find. * If more equal elements are present, the first one is returned. - * \param hashdyn The hashtable. * \param cmp Compare function called with cmp_arg as first argument and with the element to compare as a second one. - * The function should return 0 for equal elements, non-zero for different elements. + * The function should return 0 for equal elements, anything other for different elements. * \param cmp_arg Compare argument passed as first argument of the compare function. * \param hash Hash of the element to find. * \return The first element found, or 0 if none. */ tommy_inline void* tommy_hashdyn_search(tommy_hashdyn* hashdyn, tommy_search_func* cmp, const void* cmp_arg, tommy_hash_t hash) { - tommy_hashdyn_node* node = tommy_hashdyn_find(hashdyn, cmp, cmp_arg, hash); + tommy_hashdyn_node* i = tommy_hashdyn_bucket(hashdyn, hash); - return node ? node->data : 0; + while (i) { + /* we first check if the hash matches, as in the same bucket we may have multiple hash values */ + if (i->index == hash && cmp(cmp_arg, i->data) == 0) + return i->data; + i = i->next; + } + return 0; } /** * Removes an element from the hashtable. - * You must already have the address of the node to remove. - * \param hashdyn The hashtable. - * \param node The node to remove. - * \return The removed element. + * You must already have the address of the element to remove. + * \return The tommy_node::data field of the node removed. */ TOMMY_API void* tommy_hashdyn_remove_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); -/** - * Gets the first node in iteration order, or 0 if empty. - * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. - * \param hashdyn The hashtable. - * \return The first node in iteration order, or 0 if empty. - * \note This operation is expected O(1) under suitable hash distribution and - * normal dynamic resizing. In the worst case, scanning bucket contents and - * empty buckets takes O(n + b), where n is the number of elements and b is - * the number of buckets. - */ -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_head(tommy_hashdyn* hashdyn); - -/** - * Gets the last node in iteration order, or 0 if empty. - * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. - * \param hashdyn The hashtable. - * \return The last node in iteration order, or 0 if empty. - * \note This operation is expected O(1) under suitable hash distribution and - * normal dynamic resizing. In the worst case, scanning bucket contents and - * empty buckets takes O(n + b), where n is the number of elements and b is - * the number of buckets. - */ -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_tail(tommy_hashdyn* hashdyn); - -/** - * Gets the following node in iteration order, or 0 for the tail. - * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. - * \param hashdyn The hashtable. - * \param node Node currently contained in the hashtable. It must not be 0. - * \return The following node in iteration order, or 0 for the tail. - * \note This operation is expected O(1) under suitable hash distribution and - * normal dynamic resizing. In the worst case, scanning bucket contents and - * empty buckets takes O(n + b), where n is the number of elements and b is - * the number of buckets. - */ -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_next(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); - -/** - * Gets the preceding node in iteration order, or 0 for the head. - * See \ref tommy_hashdyn_iterator_validity for ordering and iterator validity. - * \param hashdyn The hashtable. - * \param node Node currently contained in the hashtable. It must not be 0. - * \return The preceding node in iteration order, or 0 for the head. - * \note This operation is expected O(1) under suitable hash distribution and - * normal dynamic resizing. In the worst case, scanning bucket contents and - * empty buckets takes O(n + b), where n is the number of elements and b is - * the number of buckets. - */ -TOMMY_API tommy_hashdyn_node* tommy_hashdyn_prev(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node); - -/** - * Updates the hash of an element already contained in the hashtable. - * The node must belong to this hashtable. The caller updates the element key - * and provides its new hash, without modifying tommy_node::index directly. - * If the hash is unchanged, the node and its position are left unchanged. - * Otherwise, the node is moved to the tail of the destination bucket, - * even if the old and new hashes identify the same bucket. - * The tommy_node::data field and the number of elements are left unchanged. - * No memory allocation, deallocation or resize is performed. - * Equal keys are allowed; no uniqueness check is performed. - * \param hashdyn The hashtable. - * \param node The node whose hash is updated. - * \param hash The new hash of the element. - * \note This operation is O(1). - */ -TOMMY_API void tommy_hashdyn_rehash_existing(tommy_hashdyn* hashdyn, tommy_hashdyn_node* node, tommy_hash_t hash); - /** * Calls the specified function for each element in the hashtable. * @@ -443,102 +230,66 @@ TOMMY_API void tommy_hashdyn_rehash_existing(tommy_hashdyn* hashdyn, tommy_hashd * * ... * - * // creates an element - * struct element* elem = malloc(sizeof(struct element)); + * // creates an object + * struct object* obj = malloc(sizeof(struct object)); * * ... * * // insert it in the hashtable - * tommy_hashdyn_insert(&hashdyn, &elem->node, elem, tommy_inthash_u32(elem->value)); + * tommy_hashdyn_insert(&hashdyn, &obj->node, obj, tommy_inthash_u32(obj->value)); * * ... * - * // deallocates all the elements iterating the hashtable + * // deallocates all the objects iterating the hashtable * tommy_hashdyn_foreach(&hashdyn, free); * * // deallocates the hashtable * tommy_hashdyn_done(&hashdyn); * \endcode - * - * If the callback frees the elements, the hashtable retains dangling - * node pointers. Call tommy_hashdyn_clear() before reusing it, or - * tommy_hashdyn_done() to release its storage. - * \param hashdyn The hashtable. - * \param func Function to call with each element. */ TOMMY_API void tommy_hashdyn_foreach(tommy_hashdyn* hashdyn, tommy_foreach_func* func); /** * Calls the specified function with an argument for each element in the hashtable. - * The iteration order and callback rules are the same as tommy_hashdyn_foreach(). - * The callback may deallocate the current element. - * Adding or removing elements from inside the callback is not allowed. - * \param hashdyn The hashtable. - * \param func Function to call with each element. - * \param arg Argument to pass to the function. */ TOMMY_API void tommy_hashdyn_foreach_arg(tommy_hashdyn* hashdyn, tommy_foreach_arg_func* func, void* arg); /** * Gets the number of elements. - * \param hashdyn The hashtable. - * \return The number of elements in the hashtable. - * \note This operation is O(1). */ -tommy_inline tommy_size_t tommy_hashdyn_count(const tommy_hashdyn* hashdyn) +tommy_inline tommy_size_t tommy_hashdyn_count(tommy_hashdyn* hashdyn) { return hashdyn->count; } -/** - * Checks whether the hashtable is empty. - * \param hashdyn The hashtable. - * \return Nonzero if the hashtable is empty, 0 otherwise. - * \note This operation is O(1). - */ -tommy_inline tommy_bool_t tommy_hashdyn_empty(const tommy_hashdyn* hashdyn) -{ - return hashdyn->count == 0; -} - -/** - * Gets the number of buckets. - * \param hashdyn The hashtable. - * \return The number of buckets. - * \note This operation is O(1). - */ -tommy_inline tommy_size_t tommy_hashdyn_bucket_count(const tommy_hashdyn* hashdyn) -{ - return hashdyn->bucket_max; -} - /** * Gets the size of allocated memory. * It includes the size of the ::tommy_hashdyn_node of the stored elements. - * \param hashdyn The hashtable. - * \return Total memory allocated by the hashtable, in bytes. */ -TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(const tommy_hashdyn* hashdyn); +TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(tommy_hashdyn* hashdyn); /** - * Transfers all elements from the hashtable into a tommy_list. + * \brief Transfers all elements from the hashtable into a tommy_list. * * Removes every element from the \p hashdyn hashtable and inserts them * into the provided \p list (at the tail), preserving the per-bucket order * but not guaranteeing any particular global order. * * After the call: - * - the hashtable is left empty and initialized, preserving its allocated buckets + * - the hashtable is left completely empty * - the target list contains all the elements that were previously in the hashtable * - * The tommy_node::data and tommy_node::index fields are left unchanged. - * * This function is useful when you need to: - * - extract all elements to process/sort them outside the hashtable + * - extract all elements to process/sort them outside the hash table * - convert the hashtable into a list for sequential iteration * - prepare for a full clear + re-insertion with different hash/ordering * - move ownership of the nodes to a list-based container * + * \note The operation is O(n) where n is the number of elements. + * \note No memory allocation is performed. + * \note The relative order of elements that were in the same bucket is preserved, + * but the order among different buckets is bucket-order dependent. + * * Typical usage pattern: * \code * tommy_list all_elements; @@ -551,15 +302,9 @@ TOMMY_API tommy_size_t tommy_hashdyn_memory_usage(const tommy_hashdyn* hashdyn); * tommy_list_sort(&all_elements, compare_by_value); * \endcode * - * \param hashdyn The hashtable to drain. - * \param list The destination list. It must be initialized and must not share - * nodes with the hashtable. Existing elements remain at the head of the list. - * \note This operation is O(b), where b is the number of buckets. - * \note No memory allocation or deallocation is performed. - * \note The relative order of elements that were in the same bucket is preserved, - * but the order among different buckets is bucket-order dependent. + * \param hashdyn The dynamic hashtable to drain + * \param list The list that will receive all the elements */ TOMMY_API void tommy_hashdyn_to_list(tommy_hashdyn* hashdyn, tommy_list* list); #endif - diff --git a/tommyds/tommylist.c b/tommyds/tommylist.c index a1d4d8ef..9f7b2102 100644 --- a/tommyds/tommylist.c +++ b/tommyds/tommylist.c @@ -7,109 +7,30 @@ /** \internal * Setup a list. */ -tommy_inline void tommy_imp_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) +tommy_inline void tommy_list_set(tommy_list* list, tommy_node* head, tommy_node* tail) { head->prev = tail; tail->next = 0; *list = head; } -TOMMY_API void tommy_list_splice_before(tommy_list* list, tommy_node* reference, tommy_list* second) -{ - tommy_node* second_head = tommy_list_head(second); - - if (!second_head) - return; - - if (reference == tommy_list_head(list)) { - /* prepend */ - tommy_list_prepend(list, second); - } else { - /* splice in the middle */ - tommy_chain_splice(reference->prev, reference, second_head, second_head->prev); - } -} - -TOMMY_API void tommy_list_splice_after(tommy_list* list, tommy_node* reference, tommy_list* second) -{ - tommy_node* second_head = tommy_list_head(second); - - if (!second_head) - return; - - if (!reference->next) { - /* append */ - tommy_list_concat(list, second); - } else { - /* splice in the middle */ - tommy_chain_splice(reference, reference->next, second_head, second_head->prev); - } -} - -TOMMY_API void tommy_list_merge(tommy_list* first, tommy_list* second, tommy_compare_func* cmp) -{ - tommy_node* second_head = tommy_list_head(second); - - if (!second_head) - return; - - tommy_node* first_head = tommy_list_head(first); - - if (!first_head) { - *first = *second; - return; - } - - /* create chains from the non-empty lists */ - tommy_chain first_chain; - tommy_chain second_chain; - first_chain.head = first_head; - first_chain.tail = first_head->prev; - second_chain.head = second_head; - second_chain.tail = second_head->prev; - - tommy_chain_merge_degenerated(&first_chain, &second_chain, cmp); - - /* restore the destination list */ - tommy_imp_list_set(first, first_chain.head, first_chain.tail); -} - -TOMMY_API void tommy_list_reverse(tommy_list* list) -{ - tommy_node* head = tommy_list_head(list); - tommy_node* node = head; - tommy_node* prev = 0; - - if (!head) - return; - - while (node) { - tommy_node* next = node->next; - node->next = prev; - node->prev = next; - prev = node; - node = next; - } - - /* the old head is the new tail */ - tommy_imp_list_set(list, prev, head); -} - TOMMY_API void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp) { + tommy_chain chain; + tommy_node* head; + if (tommy_list_empty(list)) return; - tommy_node* head = tommy_list_head(list); + head = tommy_list_head(list); /* create a chain from the list */ - tommy_chain chain; chain.head = head; chain.tail = head->prev; tommy_chain_mergesort(&chain, cmp); /* restore the list */ - tommy_imp_list_set(list, chain.head, chain.tail); + tommy_list_set(list, chain.head, chain.tail); } diff --git a/tommyds/tommylist.h b/tommyds/tommylist.h index 4de562fd..88423358 100644 --- a/tommyds/tommylist.h +++ b/tommyds/tommylist.h @@ -2,21 +2,21 @@ // Copyright (C) 2010 Andrea Mazzoleni /** \file - * Doubly linked list for collisions into hashtables. + * Double linked list for collisions into hashtables. * - * This list is a doubly linked list mainly targeted for handling collisions - * into a hashtable, but usable also as a generic list. + * This list is a **doubly** linked list mainly targeted for handling collisions + * into an hashtable, but **usable** also as a generic list. * * The main feature of this list is to require only one pointer to represent the - * list, compared to a classic implementation requiring both head and tail pointers. + * list, compared to a classic implementation requiring a head **and** a tail pointers. * This reduces the memory usage in hashtables. * - * Another feature is to support the insertion at the end of the list. This allows - * storing collisions in a stable order, meaning that equal elements preserve + * Another feature is to support the insertion at the end of the list. This **allows** to store + * collisions in a stable order. Where for stable order we mean that equal elements keep * their insertion order. * * To initialize the list, you have to call tommy_list_init(), or to simply assign - * to it 0, as an empty list is represented by the 0 value. + * to it NULL, as an empty list is represented by the NULL value. * * \code * tommy_list list; @@ -27,113 +27,51 @@ * To insert elements in the list you have to call tommy_list_insert_tail() * or tommy_list_insert_head() for each element. * In the insertion call you have to specify the address of the node and the - * address of the element. - * The address of the element is used to initialize the tommy_node::data field + * address of the object. + * The address of the object is used to initialize the tommy_node::data field * of the node. * * \code - * struct element { + * struct object { * int value; * // other fields * tommy_node node; * }; * - * struct element* elem = malloc(sizeof(struct element)); // creates the element + * struct object* obj = malloc(sizeof(struct object)); // creates the object * - * elem->value = ...; // initializes the element + * obj->value = ...; // initializes the object * - * tommy_list_insert_tail(&list, &elem->node, elem); // inserts the element + * tommy_list_insert_tail(&list, &obj->node, obj); // inserts the object * \endcode * * To iterate over all the elements in the list you have to call * tommy_list_head() to get the head of the list and follow the - * tommy_node::next pointer until 0. + * tommy_node::next pointer until NULL. * * \code * tommy_node* i = tommy_list_head(&list); * while (i) { - * struct element* elem = i->data; // gets the element pointer + * struct object* obj = i->data; // gets the object pointer * - * printf("%d\n", elem->value); // process the element + * printf("%d\n", obj->value); // process the object * * i = i->next; // go to the next element * } * \endcode * - * To iterate backward, start at tommy_list_tail() and use tommy_list_prev(). - * Do not follow tommy_node::prev directly past the head, as it points to the tail. - * - * \code - * tommy_node* i = tommy_list_tail(&list); - * while (i) { - * struct element* elem = i->data; - * - * printf("%d\n", elem->value); - * - * i = tommy_list_prev(&list, i); - * } - * \endcode - * - * \anchor tommy_list_iterator_validity - * Iterators are pointers to nodes contained in the list. - * Inserting elements or removing other nodes between iterator calls does not - * invalidate pointers to nodes that remain in the list. - * Subsequent next/prev calls use the current contents and ordering; - * inserted elements may become visible and removed elements are skipped. - * - * Removing the pointed node invalidates its use as an iterator, even if - * its storage has not been freed. To continue traversal after removing - * the current node, obtain the next or previous node before removal. - * - * \code - * tommy_node* i = tommy_list_head(&list); - * while (i) { - * tommy_node* next = i->next; // obtain the next node before removal - * - * if (condition) { - * tommy_list_remove_existing(&list, i); - * free(i->data); // safe to deallocate - * } - * - * i = next; - * } - * \endcode - * - * Concurrent traversal and modification require external synchronization. - * - * To reorder existing elements, use tommy_list_move_head(), tommy_list_move_tail() - * or tommy_list_reverse(). To split a list or transfer elements between lists, - * use tommy_list_split(), tommy_list_prepend(), tommy_list_splice_before() - * and tommy_list_splice_after(). - * These operations preserve the element pointers and do not allocate memory. - * - * \code - * tommy_list second; - * tommy_list_init(&second); - * - * // transfers node and all following nodes to second - * tommy_list_split(&list, node, &second); - * - * // appends the transferred nodes back to list - * tommy_list_concat(&list, &second); - * \endcode - * - * Two lists sorted with the same comparison function can be merged with - * tommy_list_merge(). The merge is stable. - * * To destroy the list you have to remove all the elements, - * as the list is completely in-place and it doesn't allocate memory. + * as the list is completely inplace and it doesn't allocate memory. * This can be done with the tommy_list_foreach() function. * * \code - * // deallocates all the elements iterating the list + * // deallocates all the objects iterating the list * tommy_list_foreach(&list, free); - * tommy_list_clear(&list); // resets the list before reuse * \endcode */ -#ifndef TOMMY_LIST_H -#define TOMMY_LIST_H +#ifndef __TOMMYLIST_H +#define __TOMMYLIST_H #include "tommytypes.h" @@ -145,72 +83,18 @@ */ typedef tommy_node* tommy_list; -/** - * Temporary list builder with an embedded sentinel. - * Initialize with tommy_builder_init() and finish with tommy_list_concat_builder(). - * Keep the builder at the same address until finalization, as node links - * refer to its sentinel. - */ -typedef struct tommy_builder_struct { - tommy_node dummy; /**< Sentinel excluded from the resulting list. */ -} tommy_builder; - -/** - * Initializes an empty builder in O(1). - * \param builder Builder to initialize in its final location. - * \return Initial tail, pointing to the builder's sentinel. - */ -tommy_inline tommy_node* tommy_builder_init(tommy_builder* builder) -{ - builder->dummy.next = 0; - return &builder->dummy; -} - -/** - * Appends a non-empty chain to a builder in O(1), preserving element order. - * \param tail Current tail, initially returned by tommy_builder_init(). - * \param head Head of the non-empty source chain. - * \param last Tail of the source chain. For a single node, pass head again. - * The chains must not share nodes. Inner links must be valid; - * head's prev and last's next may be undefined. - * \return Updated tail to pass to the next append or tommy_list_concat_builder(). - */ -tommy_inline tommy_node* tommy_builder_concat(tommy_node* tail, tommy_node* head, tommy_node* last) -{ - head->prev = tail; - tail->next = head; - return last; -} - /** * Initializes the list. - * The list is completely in-place, so it doesn't need to be deinitialized. - * \param list The list. + * The list is completely inplace, so it doesn't need to be deinitialized. */ tommy_inline void tommy_list_init(tommy_list* list) { *list = 0; } -/** - * Removes all elements. - * The list remains initialized and can be reused immediately. - * Elements are not freed and nodes are not accessed or modified. - * Their links must not be used to traverse the previous contents. - * You can call this function after tommy_list_foreach() has freed the elements. - * \param list The list. - * \note This operation is O(1). - */ -tommy_inline void tommy_list_clear(tommy_list* list) -{ - *list = 0; -} - /** * Gets the head of the list. - * \param list The list. - * \return The head node, or 0 if the list is empty. - * \note This operation is O(1). + * \return The head node. For empty lists 0 is returned. */ tommy_inline tommy_node* tommy_list_head(tommy_list* list) { @@ -219,9 +103,7 @@ tommy_inline tommy_node* tommy_list_head(tommy_list* list) /** * Gets the tail of the list. - * \param list The list. - * \return The tail node, or 0 if the list is empty. - * \note This operation is O(1). + * \return The tail node. For empty lists 0 is returned. */ tommy_inline tommy_node* tommy_list_tail(tommy_list* list) { @@ -233,27 +115,9 @@ tommy_inline tommy_node* tommy_list_tail(tommy_list* list) return head->prev; } -/** - * Gets the previous node in the list. - * Unlike tommy_node::prev, this function returns 0 at the head of the list, - * allowing backward iteration starting from tommy_list_tail(). - * See \ref tommy_list_iterator_validity for iterator validity across insertions and removals. - * \param list The list. - * \param node The current node. The node must be in the list. - * \return The previous node, or 0 if the current node is the head. - * \note This operation is O(1). - */ -tommy_inline tommy_node* tommy_list_prev(tommy_list* list, tommy_node* node) -{ - if (node == tommy_list_head(list)) - return 0; - - return node->prev; -} - /** \internal * Creates a new list with a single element. - * \param list The list. + * \param list The list to initialize. * \param node The node to insert. */ tommy_inline void tommy_list_insert_first(tommy_list* list, tommy_node* node) @@ -270,7 +134,7 @@ tommy_inline void tommy_list_insert_first(tommy_list* list, tommy_node* node) /** \internal * Inserts an element at the head of a non-empty list. * The element is inserted at the head of the list. The list cannot be empty. - * \param list The list. + * \param list The list. The list cannot be empty. * \param node The node to insert. */ tommy_inline void tommy_list_insert_head_not_empty(tommy_list* list, tommy_node* node) @@ -307,8 +171,8 @@ tommy_inline void tommy_list_insert_tail_not_empty(tommy_node* head, tommy_node* /** * Inserts an element at the head of a list. * \param list The list. - * \param node Pointer to the node embedded into the element to insert. - * \param data Pointer to the element to insert. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. */ tommy_inline void tommy_list_insert_head(tommy_list* list, tommy_node* node, void* data) { @@ -325,8 +189,8 @@ tommy_inline void tommy_list_insert_head(tommy_list* list, tommy_node* node, voi /** * Inserts an element at the tail of a list. * \param list The list. - * \param node Pointer to the node embedded into the element to insert. - * \param data Pointer to the element to insert. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. */ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, void* data) { @@ -344,8 +208,8 @@ tommy_inline void tommy_list_insert_tail(tommy_list* list, tommy_node* node, voi * Inserts an element before the specified reference node. * \param list The list. * \param reference The reference node. The new node will be inserted before this node. The reference node must be in the list. - * \param node Pointer to the node embedded into the element to insert. - * \param data Pointer to the element to insert. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. */ tommy_inline void tommy_list_insert_before(tommy_list* list, tommy_node* reference, tommy_node* node, void* data) { @@ -371,8 +235,8 @@ tommy_inline void tommy_list_insert_before(tommy_list* list, tommy_node* referen * Inserts an element after the specified reference node. * \param list The list. * \param reference The reference node. The new node will be inserted after this node. The reference node must be in the list. - * \param node Pointer to the node embedded into the element to insert. - * \param data Pointer to the element to insert. + * \param node The node to insert. + * \param data The object containing the node. It's used to set the tommy_node::data field of the node. */ tommy_inline void tommy_list_insert_after(tommy_list* list, tommy_node* reference, tommy_node* node, void* data) { @@ -396,12 +260,12 @@ tommy_inline void tommy_list_insert_after(tommy_list* list, tommy_node* referenc /** * Removes an element from the list. - * You must already have the address of the node to remove. - * \param list The list. - * \param node The node to remove. - * \return The removed element. + * You must already have the address of the element to remove. * \note The node content is left unchanged, including the tommy_node::next - * and tommy_node::prev fields that still contain pointers in the list. + * and tommy_node::prev fields that still contain pointers **in** the list. + * \param list The list. + * \param node The node to remove. The node must be in the list. + * \return The tommy_node::data field of the node removed. */ tommy_inline void* tommy_list_remove_existing(tommy_list* list, tommy_node* node) { @@ -423,167 +287,32 @@ tommy_inline void* tommy_list_remove_existing(tommy_list* list, tommy_node* node } /** - * Removes the element at the head of the list. - * \param list The list. - * \return The removed element, or 0 if the list is empty. - */ -tommy_inline void* tommy_list_remove_head(tommy_list* list) -{ - tommy_node* node = tommy_list_head(list); - - if (!node) - return 0; - - /* the new head must retain the tail pointer in the "circular" prev list */ - if (node->next) - node->next->prev = node->prev; - - *list = node->next; - - return node->data; -} - -/** - * Removes the element at the tail of the list. - * \param list The list. - * \return The removed element, or 0 if the list is empty. - */ -tommy_inline void* tommy_list_remove_tail(tommy_list* list) -{ - tommy_node* head = tommy_list_head(list); - - if (!head) - return 0; - - tommy_node* node = head->prev; - - /* a single node is also the head, and its links must remain unchanged */ - if (node == head) { - *list = 0; - } else { - head->prev = node->prev; - node->prev->next = 0; - } - - return node->data; -} - -/** - * Moves an existing node to the head of the list. - * If the node is already at the head, nothing is done. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param list The list. - * \param node The node to move. The node must be in the list. - */ -tommy_inline void tommy_list_move_head(tommy_list* list, tommy_node* node) -{ - if (node == tommy_list_head(list)) - return; - - tommy_list_remove_existing(list, node); - tommy_list_insert_head_not_empty(list, node); -} - -/** - * Moves an existing node to the tail of the list. - * If the node is already at the tail, nothing is done. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param list The list. - * \param node The node to move. The node must be in the list. - */ -tommy_inline void tommy_list_move_tail(tommy_list* list, tommy_node* node) -{ - if (node->next == 0) - return; - - tommy_list_remove_existing(list, node); - tommy_list_insert_tail_not_empty(tommy_list_head(list), node); -} - -/** - * Splits a list before the specified node. - * The list retains the nodes preceding the specified node. The second list - * receives the specified node and all following nodes. The order of the elements is preserved. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param list The list to split. - * \param node The first node to transfer. The node must be in the list, - * or 0 to leave list unchanged and set second to 0. - * Specifying the head transfers the whole list. - * \param second Output list receiving node and all following nodes. - * It must be distinct from list and need not be initialized. - * Its previous value is overwritten. Any previously referenced nodes - * are left unchanged and are not deallocated. - * \note This operation is O(1). - */ -tommy_inline void tommy_list_split(tommy_list* list, tommy_node* node, tommy_list* second) -{ - tommy_node* head = tommy_list_head(list); - tommy_node* tail; - tommy_node* before; - - if (!node) { - *second = 0; - return; - } - - if (node == head) { - *second = *list; - *list = 0; - return; - } - - tail = head->prev; - before = node->prev; - - /* terminate the retained list */ - head->prev = before; - before->next = 0; - - /* create the list of transferred nodes */ - node->prev = tail; - *second = node; -} - -/** - * Exchanges the contents of two lists. - * The lists must not share nodes. Nodes and elements are not accessed or modified. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * Passing the same list twice has no effect. - * \param first The first list. - * \param second The second list. - * \note This operation is O(1). - */ -tommy_inline void tommy_list_swap(tommy_list* first, tommy_list* second) -{ - tommy_list tmp = *first; - *first = *second; - *second = tmp; -} - -/** - * Concatenates two lists. - * The second list is concatenated to the first list. + * Concats two lists. + * The second list is concatenated at the first list. * \param first The first list. * \param second The second list. After this call the list content is undefined, * and you should not use it anymore. - * \note This operation is O(1). */ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second) { + tommy_node* first_head; + tommy_node* first_tail; + tommy_node* second_head; + /* if the second is empty, nothing to do */ - tommy_node* second_head = tommy_list_head(second); + second_head = tommy_list_head(second); if (second_head == 0) return; /* if the first is empty, copy the second */ - tommy_node* first_head = tommy_list_head(first); + first_head = tommy_list_head(first); if (first_head == 0) { *first = *second; return; } /* tail of the first list */ - tommy_node* first_tail = first_head->prev; + first_tail = first_head->prev; /* set the "circular" prev list */ first_head->prev = second_head->prev; @@ -593,136 +322,32 @@ tommy_inline void tommy_list_concat(tommy_list* first, tommy_list* second) first_tail->next = second_head; } -/** - * Appends a builder's chain to a list, preserving element order and restoring list links. - * \param list Destination list, possibly empty. It must not share nodes with the builder. - * \param builder Initialized builder, possibly empty. Its contents are consumed; - * reinitialize it before reuse. - * \param tail Current tail returned by the builder helpers, ignored when empty. - * \note This operation is O(1). An empty builder has no effect. - */ -tommy_inline void tommy_list_concat_builder(tommy_list* list, const tommy_builder* builder, tommy_node* tail) -{ - tommy_node* head = builder->dummy.next; - - if (!head) - return; - - tommy_node* first = *list; - - /* replace the sentinel link so no list node refers to the temporary builder */ - if (first) { - tommy_node* first_tail = first->prev; - first_tail->next = head; - head->prev = first_tail; - first->prev = tail; - } else { - *list = head; - head->prev = tail; - } - - tail->next = 0; -} - -/** - * Prepends a list to another list. - * The second list is prepended to the first list. - * \param first The first list. - * \param second The second list. After this call the list content is undefined, - * and you should not use it anymore. - * \note This operation is O(1). - */ -tommy_inline void tommy_list_prepend(tommy_list* first, tommy_list* second) -{ - tommy_list_concat(second, first); - *first = *second; -} - -/** - * Transfers all elements of a list before the specified reference node. - * The order of the transferred elements is preserved. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param list The destination list. - * \param reference The node before which to insert. It must not be 0 and must be in the destination list. - * \param second The source list. After this call the list content is undefined, - * and you should not use it anymore. - * The lists must be distinct and must not share nodes. - * \note This operation is O(1). An empty source list has no effect. - */ -TOMMY_API void tommy_list_splice_before(tommy_list* list, tommy_node* reference, tommy_list* second); - -/** - * Transfers all elements of a list after the specified reference node. - * The order of the transferred elements is preserved. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param list The destination list. - * \param reference The node after which to insert. It must not be 0 and must be in the destination list. - * \param second The source list. After this call the list content is undefined, - * and you should not use it anymore. - * The lists must be distinct and must not share nodes. - * \note This operation is O(1). An empty source list has no effect. - */ -TOMMY_API void tommy_list_splice_after(tommy_list* list, tommy_node* reference, tommy_list* second); - -/** - * Merges two sorted lists. - * Both lists must be sorted using the same comparison function. - * The merge is stable: equal elements from the first list precede those from - * the second list, and the relative order within each list is preserved. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param first The destination list, receiving the sorted result. - * \param second The source list. After this call the list content is undefined, - * and you should not use it anymore. - * The lists must be distinct and must not share nodes. - * \param cmp Compare function called with two elements. - * The function should return <0 if the first element is less than the second, - * ==0 if equal, and >0 if greater. It must not modify the lists. - * \note This operation is O(n+m), or O(1) when one list is empty or the ranges - * can be concatenated directly. It uses O(1) auxiliary space. - */ -TOMMY_API void tommy_list_merge(tommy_list* first, tommy_list* second, tommy_compare_func* cmp); - -/** - * Reverses the order of the elements in a list. - * Empty lists and lists containing one element are left unchanged. - * The tommy_node::data and tommy_node::index fields are left unchanged. - * \param list The list. - * \note This operation is O(n) and uses O(1) auxiliary space. - */ -TOMMY_API void tommy_list_reverse(tommy_list* list); - /** * Sorts a list. - * It's a stable merge sort with O(n log n) worst complexity. - * It's faster on degenerate cases like partially ordered lists. - * \param list The list. + * It's a stable merge sort with O(N*log(N)) worst complexity. + * It's faster on degenerated cases like partially ordered lists. * \param cmp Compare function called with two elements. * The function should return <0 if the first element is less than the second, ==0 if equal, and >0 if greater. - * \note This operation is O(n log n) in the worst case. */ TOMMY_API void tommy_list_sort(tommy_list* list, tommy_compare_func* cmp); /** - * Checks whether the list is empty. - * \param list The list. - * \return Nonzero if the list is empty, 0 otherwise. - * \note This operation is O(1). + * Checks if empty. + * \return If the list is empty. */ -tommy_inline tommy_bool_t tommy_list_empty(const tommy_list* list) +tommy_inline tommy_bool_t tommy_list_empty(tommy_list* list) { - return *list == 0; + return tommy_list_head(list) == 0; } /** * Gets the number of elements. - * \param list The list. - * \return The number of elements in the list. * \note This operation is O(n). */ -tommy_inline tommy_size_t tommy_list_count(const tommy_list* list) +tommy_inline tommy_size_t tommy_list_count(tommy_list* list) { tommy_size_t count = 0; - const tommy_node* i = *list; + tommy_node* i = tommy_list_head(list); while (i) { ++count; @@ -746,26 +371,19 @@ tommy_inline tommy_size_t tommy_list_count(const tommy_list* list) * * ... * - * // creates an element - * struct element* elem = malloc(sizeof(struct element)); + * // creates an object + * struct object* obj = malloc(sizeof(struct object)); * * ... * * // insert it in the list - * tommy_list_insert_tail(&list, &elem->node, elem); + * tommy_list_insert_tail(&list, &obj->node, obj); * * ... * - * // deallocates all the elements iterating the list + * // deallocates all the objects iterating the list * tommy_list_foreach(&list, free); - * tommy_list_clear(&list); // resets the list before reuse * \endcode - * - * If the callback frees the elements, the list retains dangling node - * pointers. Call tommy_list_clear() or tommy_list_init() before reusing it. - * The list has no storage to release. - * \param list The list. - * \param func Function to call with each element. */ tommy_inline void tommy_list_foreach(tommy_list* list, tommy_foreach_func* func) { @@ -780,12 +398,6 @@ tommy_inline void tommy_list_foreach(tommy_list* list, tommy_foreach_func* func) /** * Calls the specified function with an argument for each element in the list. - * The iteration order and callback rules are the same as tommy_list_foreach(). - * The callback may deallocate the current element. - * Adding or removing elements from inside the callback is not allowed. - * \param list The list. - * \param func Function to call with each element. - * \param arg Argument to pass to the function. */ tommy_inline void tommy_list_foreach_arg(tommy_list* list, tommy_foreach_arg_func* func, void* arg) { @@ -799,4 +411,3 @@ tommy_inline void tommy_list_foreach_arg(tommy_list* list, tommy_foreach_arg_fun } #endif - diff --git a/tommyds/tommytree.c b/tommyds/tommytree.c index 069401be..7f3fcd19 100644 --- a/tommyds/tommytree.c +++ b/tommyds/tommytree.c @@ -14,7 +14,7 @@ #define TOMMY_TREE_BALANCE_LEFT ((tommy_size_t)1) #define TOMMY_TREE_BALANCE_RIGHT ((tommy_size_t)2) -tommy_inline int tommy_imp_tree_balance_get(tommy_tree_node* node) +tommy_inline int tommy_tree_balance_get(tommy_tree_node* node) { tommy_size_t balance = node->index & TOMMY_TREE_BALANCE_MASK; @@ -25,7 +25,7 @@ tommy_inline int tommy_imp_tree_balance_get(tommy_tree_node* node) return 0; } -tommy_inline void tommy_imp_tree_balance_set(tommy_tree_node* node, int balance) +tommy_inline void tommy_tree_balance_set(tommy_tree_node* node, int balance) { tommy_size_t value = 0; @@ -37,12 +37,12 @@ tommy_inline void tommy_imp_tree_balance_set(tommy_tree_node* node, int balance) node->index = (node->index & ~TOMMY_TREE_BALANCE_MASK) | value; } -tommy_inline void tommy_imp_tree_parent_set(tommy_tree_node* node, tommy_tree_node* parent) +tommy_inline void tommy_tree_parent_set(tommy_tree_node* node, tommy_tree_node* parent) { node->index = (tommy_size_t)(tommy_uintptr_t)parent | (node->index & TOMMY_TREE_BALANCE_MASK); } -tommy_inline void tommy_imp_tree_replace(tommy_tree* tree, tommy_tree_node* root, tommy_tree_node* node) +tommy_inline void tommy_tree_replace(tommy_tree* tree, tommy_tree_node* root, tommy_tree_node* node) { tommy_tree_node* parent = tommy_tree_parent(root); @@ -55,45 +55,45 @@ tommy_inline void tommy_imp_tree_replace(tommy_tree* tree, tommy_tree_node* root } if (node) - tommy_imp_tree_parent_set(node, parent); + tommy_tree_parent_set(node, parent); } -tommy_inline tommy_tree_node* tommy_imp_tree_rotate_left(tommy_tree* tree, tommy_tree_node* root) +tommy_inline tommy_tree_node* tommy_tree_rotate_left(tommy_tree* tree, tommy_tree_node* root) { tommy_tree_node* next = root->next; - tommy_imp_tree_replace(tree, root, next); + tommy_tree_replace(tree, root, next); root->next = next->prev; if (root->next) - tommy_imp_tree_parent_set(root->next, root); + tommy_tree_parent_set(root->next, root); next->prev = root; - tommy_imp_tree_parent_set(root, next); + tommy_tree_parent_set(root, next); return next; } -tommy_inline tommy_tree_node* tommy_imp_tree_rotate_right(tommy_tree* tree, tommy_tree_node* root) +tommy_inline tommy_tree_node* tommy_tree_rotate_right(tommy_tree* tree, tommy_tree_node* root) { tommy_tree_node* prev = root->prev; - tommy_imp_tree_replace(tree, root, prev); + tommy_tree_replace(tree, root, prev); root->prev = prev->next; if (root->prev) - tommy_imp_tree_parent_set(root->prev, root); + tommy_tree_parent_set(root->prev, root); prev->next = root; - tommy_imp_tree_parent_set(root, prev); + tommy_tree_parent_set(root, prev); return prev; } -tommy_inline void tommy_imp_tree_insert_balance(tommy_tree* tree, tommy_tree_node* node) +tommy_inline void tommy_tree_insert_balance(tommy_tree* tree, tommy_tree_node* node) { tommy_tree_node* parent = tommy_tree_parent(node); while (parent) { - int balance = tommy_imp_tree_balance_get(parent); + int balance = tommy_tree_balance_get(parent); if (node == parent->prev) --balance; @@ -102,12 +102,12 @@ tommy_inline void tommy_imp_tree_insert_balance(tommy_tree* tree, tommy_tree_nod } if (balance == 0) { - tommy_imp_tree_balance_set(parent, balance); + tommy_tree_balance_set(parent, balance); return; } if (balance >= -1 && balance <= 1) { - tommy_imp_tree_balance_set(parent, balance); + tommy_tree_balance_set(parent, balance); node = parent; parent = tommy_tree_parent(parent); continue; @@ -115,39 +115,39 @@ tommy_inline void tommy_imp_tree_insert_balance(tommy_tree* tree, tommy_tree_nod if (balance == -2) { tommy_tree_node* left = parent->prev; - int left_balance = tommy_imp_tree_balance_get(left); + int left_balance = tommy_tree_balance_get(left); if (left_balance < 0) { - tommy_imp_tree_rotate_right(tree, parent); - tommy_imp_tree_balance_set(parent, 0); - tommy_imp_tree_balance_set(left, 0); + tommy_tree_rotate_right(tree, parent); + tommy_tree_balance_set(parent, 0); + tommy_tree_balance_set(left, 0); } else { tommy_tree_node* middle = left->next; - int middle_balance = tommy_imp_tree_balance_get(middle); + int middle_balance = tommy_tree_balance_get(middle); - tommy_imp_tree_rotate_left(tree, left); - tommy_imp_tree_rotate_right(tree, parent); - tommy_imp_tree_balance_set(parent, middle_balance < 0 ? 1 : 0); - tommy_imp_tree_balance_set(left, middle_balance > 0 ? -1 : 0); - tommy_imp_tree_balance_set(middle, 0); + tommy_tree_rotate_left(tree, left); + tommy_tree_rotate_right(tree, parent); + tommy_tree_balance_set(parent, middle_balance < 0 ? 1 : 0); + tommy_tree_balance_set(left, middle_balance > 0 ? -1 : 0); + tommy_tree_balance_set(middle, 0); } } else { tommy_tree_node* right = parent->next; - int right_balance = tommy_imp_tree_balance_get(right); + int right_balance = tommy_tree_balance_get(right); if (right_balance > 0) { - tommy_imp_tree_rotate_left(tree, parent); - tommy_imp_tree_balance_set(parent, 0); - tommy_imp_tree_balance_set(right, 0); + tommy_tree_rotate_left(tree, parent); + tommy_tree_balance_set(parent, 0); + tommy_tree_balance_set(right, 0); } else { tommy_tree_node* middle = right->prev; - int middle_balance = tommy_imp_tree_balance_get(middle); + int middle_balance = tommy_tree_balance_get(middle); - tommy_imp_tree_rotate_right(tree, right); - tommy_imp_tree_rotate_left(tree, parent); - tommy_imp_tree_balance_set(parent, middle_balance > 0 ? -1 : 0); - tommy_imp_tree_balance_set(right, middle_balance < 0 ? 1 : 0); - tommy_imp_tree_balance_set(middle, 0); + tommy_tree_rotate_right(tree, right); + tommy_tree_rotate_left(tree, parent); + tommy_tree_balance_set(parent, middle_balance > 0 ? -1 : 0); + tommy_tree_balance_set(right, middle_balance < 0 ? 1 : 0); + tommy_tree_balance_set(middle, 0); } } @@ -155,59 +155,41 @@ tommy_inline void tommy_imp_tree_insert_balance(tommy_tree* tree, tommy_tree_nod } } -tommy_inline void* tommy_imp_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data, tommy_bool_t unique) +TOMMY_API void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) { tommy_tree_node* parent = 0; - tommy_tree_node* existing = 0; tommy_tree_node** link = &tree->root; - /* cache the comparator to avoid reloading it after each comparison */ - tommy_compare_func* cmp_func = tree->cmp; - - while (*link) { - /* comparators must not change the tree, so keep the node across the call */ - parent = *link; - int cmp = cmp_func(data, parent->data); - - if (cmp < 0) - link = &parent->prev; - else if (cmp == 0 && unique) { - existing = parent; - link = &parent->prev; - } else { - /* placing equal keys after existing ones preserves insertion order through rotations */ - link = &parent->next; - } - } - - if (existing) - return existing->data; node->data = data; node->prev = 0; node->next = 0; + node->index = 0; + + while (*link) { + int cmp = tree->cmp(data, (*link)->data); + + parent = *link; + if (cmp < 0) + link = &parent->prev; + else if (cmp > 0) + link = &parent->next; + else + return parent->data; + } + node->index = (tommy_size_t)(tommy_uintptr_t)parent; *link = node; ++tree->count; - tommy_imp_tree_insert_balance(tree, node); + tommy_tree_insert_balance(tree, node); return node->data; } -TOMMY_API void tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) -{ - tommy_imp_tree_insert(tree, node, data, 0); -} - -TOMMY_API void* tommy_tree_insert_unique(tommy_tree* tree, tommy_tree_node* node, void* data) -{ - return tommy_imp_tree_insert(tree, node, data, 1); -} - -tommy_inline void tommy_imp_tree_remove_balance(tommy_tree* tree, tommy_tree_node* node, tommy_bool_t left_shrunk) +tommy_inline void tommy_tree_remove_balance(tommy_tree* tree, tommy_tree_node* node, tommy_bool_t left_shrunk) { while (node) { - int balance = tommy_imp_tree_balance_get(node); + int balance = tommy_tree_balance_get(node); if (left_shrunk) ++balance; @@ -215,13 +197,15 @@ tommy_inline void tommy_imp_tree_remove_balance(tommy_tree* tree, tommy_tree_nod --balance; if (balance == -1 || balance == 1) { - tommy_imp_tree_balance_set(node, balance); + tommy_tree_balance_set(node, balance); return; } if (balance == 0) { - tommy_imp_tree_balance_set(node, balance); - tommy_tree_node* parent = tommy_tree_parent(node); + tommy_tree_node* parent; + + tommy_tree_balance_set(node, balance); + parent = tommy_tree_parent(node); if (!parent) return; left_shrunk = node == parent->prev; @@ -231,27 +215,27 @@ tommy_inline void tommy_imp_tree_remove_balance(tommy_tree* tree, tommy_tree_nod if (balance == -2) { tommy_tree_node* left = node->prev; - int left_balance = tommy_imp_tree_balance_get(left); + int left_balance = tommy_tree_balance_get(left); tommy_tree_node* root; if (left_balance <= 0) { - root = tommy_imp_tree_rotate_right(tree, node); + root = tommy_tree_rotate_right(tree, node); if (left_balance == 0) { - tommy_imp_tree_balance_set(node, -1); - tommy_imp_tree_balance_set(left, 1); + tommy_tree_balance_set(node, -1); + tommy_tree_balance_set(left, 1); return; } - tommy_imp_tree_balance_set(node, 0); - tommy_imp_tree_balance_set(left, 0); + tommy_tree_balance_set(node, 0); + tommy_tree_balance_set(left, 0); } else { tommy_tree_node* middle = left->next; - int middle_balance = tommy_imp_tree_balance_get(middle); + int middle_balance = tommy_tree_balance_get(middle); - tommy_imp_tree_rotate_left(tree, left); - root = tommy_imp_tree_rotate_right(tree, node); - tommy_imp_tree_balance_set(node, middle_balance < 0 ? 1 : 0); - tommy_imp_tree_balance_set(left, middle_balance > 0 ? -1 : 0); - tommy_imp_tree_balance_set(middle, 0); + tommy_tree_rotate_left(tree, left); + root = tommy_tree_rotate_right(tree, node); + tommy_tree_balance_set(node, middle_balance < 0 ? 1 : 0); + tommy_tree_balance_set(left, middle_balance > 0 ? -1 : 0); + tommy_tree_balance_set(middle, 0); } node = tommy_tree_parent(root); @@ -263,27 +247,27 @@ tommy_inline void tommy_imp_tree_remove_balance(tommy_tree* tree, tommy_tree_nod { tommy_tree_node* right = node->next; - int right_balance = tommy_imp_tree_balance_get(right); + int right_balance = tommy_tree_balance_get(right); tommy_tree_node* root; if (right_balance >= 0) { - root = tommy_imp_tree_rotate_left(tree, node); + root = tommy_tree_rotate_left(tree, node); if (right_balance == 0) { - tommy_imp_tree_balance_set(node, 1); - tommy_imp_tree_balance_set(right, -1); + tommy_tree_balance_set(node, 1); + tommy_tree_balance_set(right, -1); return; } - tommy_imp_tree_balance_set(node, 0); - tommy_imp_tree_balance_set(right, 0); + tommy_tree_balance_set(node, 0); + tommy_tree_balance_set(right, 0); } else { tommy_tree_node* middle = right->prev; - int middle_balance = tommy_imp_tree_balance_get(middle); + int middle_balance = tommy_tree_balance_get(middle); - tommy_imp_tree_rotate_right(tree, right); - root = tommy_imp_tree_rotate_left(tree, node); - tommy_imp_tree_balance_set(node, middle_balance > 0 ? -1 : 0); - tommy_imp_tree_balance_set(right, middle_balance < 0 ? 1 : 0); - tommy_imp_tree_balance_set(middle, 0); + tommy_tree_rotate_right(tree, right); + root = tommy_tree_rotate_left(tree, node); + tommy_tree_balance_set(node, middle_balance > 0 ? -1 : 0); + tommy_tree_balance_set(right, middle_balance < 0 ? 1 : 0); + tommy_tree_balance_set(middle, 0); } node = tommy_tree_parent(root); @@ -305,7 +289,7 @@ TOMMY_API void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* no parent = tommy_tree_parent(node); left_shrunk = parent && node == parent->prev; - tommy_imp_tree_replace(tree, node, child); + tommy_tree_replace(tree, node, child); } else { tommy_tree_node* next = node->next; @@ -314,106 +298,67 @@ TOMMY_API void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* no parent = tommy_tree_parent(next); if (parent == node) { - tommy_imp_tree_replace(tree, node, next); + tommy_tree_replace(tree, node, next); next->prev = node->prev; - tommy_imp_tree_parent_set(next->prev, next); - tommy_imp_tree_balance_set(next, tommy_imp_tree_balance_get(node)); + tommy_tree_parent_set(next->prev, next); + tommy_tree_balance_set(next, tommy_tree_balance_get(node)); parent = next; left_shrunk = 0; } else { parent->prev = next->next; if (parent->prev) - tommy_imp_tree_parent_set(parent->prev, parent); + tommy_tree_parent_set(parent->prev, parent); - tommy_imp_tree_replace(tree, node, next); + tommy_tree_replace(tree, node, next); next->prev = node->prev; next->next = node->next; - tommy_imp_tree_parent_set(next->prev, next); - tommy_imp_tree_parent_set(next->next, next); - tommy_imp_tree_balance_set(next, tommy_imp_tree_balance_get(node)); + tommy_tree_parent_set(next->prev, next); + tommy_tree_parent_set(next->next, next); + tommy_tree_balance_set(next, tommy_tree_balance_get(node)); left_shrunk = 1; } } --tree->count; if (parent) - tommy_imp_tree_remove_balance(tree, parent, left_shrunk); + tommy_tree_remove_balance(tree, parent, left_shrunk); return data; } -TOMMY_API void tommy_tree_to_list(tommy_tree* tree, tommy_list* list) +TOMMY_API void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func) { - /* AVL height is less than twice the bit width of its node count */ - tommy_tree_node* stack[2 * TOMMY_SIZE_BIT]; - tommy_size_t depth = 0; - tommy_tree_node* node = tree->root; - tommy_builder builder; - tommy_node* builder_tail = tommy_builder_init(&builder); + while (root) { + tommy_tree_node* next; - while (node || depth) { - while (node) { - stack[depth] = node; - ++depth; - node = node->prev; - } + tommy_tree_foreach_node(root->prev, func); - node = stack[--depth]; - /* chain appending overwrites child links, so keep the right subtree reachable */ - tommy_tree_node* next = node->next; - builder_tail = tommy_builder_concat(builder_tail, node, node); - node = next; - } + /* make a copy in case func is free() */ + next = root->next; - tommy_list_concat_builder(list, &builder, builder_tail); - tommy_tree_clear(tree); -} + func(root->data); -TOMMY_API void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) -{ - /* AVL height is less than twice the bit width of its node count */ - tommy_tree_node* stack[2 * TOMMY_SIZE_BIT]; - tommy_size_t depth = 0; - tommy_tree_node* node = tree->root; - - while (node || depth) { - while (node) { - stack[depth] = node; - ++depth; - node = node->prev; - } - - node = stack[--depth]; - /* save the next subtree before the callback can free this node */ - tommy_tree_node* next = node->next; - func(node->data); - node = next; + root = next; } } -TOMMY_API void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) +TOMMY_API void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg) { - /* AVL height is less than twice the bit width of its node count */ - tommy_tree_node* stack[2 * TOMMY_SIZE_BIT]; - tommy_size_t depth = 0; - tommy_tree_node* node = tree->root; + while (root) { + tommy_tree_node* next; - while (node || depth) { - while (node) { - stack[depth] = node; - ++depth; - node = node->prev; - } + tommy_tree_foreach_arg_node(root->prev, func, arg); - node = stack[--depth]; - /* save the next subtree before the callback can free this node */ - tommy_tree_node* next = node->next; - func(arg, node->data); - node = next; + /* make a copy in case func is free() */ + next = root->next; + + func(arg, root->data); + + root = next; } } -TOMMY_API tommy_size_t tommy_tree_memory_usage(const tommy_tree* tree) +TOMMY_API tommy_size_t tommy_tree_memory_usage(tommy_tree* tree) { return tommy_tree_count(tree) * sizeof(tommy_tree_node); } diff --git a/tommyds/tommytree.h b/tommyds/tommytree.h index f2ff9f99..9bae8611 100644 --- a/tommyds/tommytree.h +++ b/tommyds/tommytree.h @@ -7,8 +7,7 @@ * This tree is a standard AVL tree implementation that stores elements in the * order defined by the comparison function. * - * Duplicate elements are allowed. They are visited in insertion order. - * Use tommy_tree_insert_unique() to reject duplicate keys. + * As a difference from other tommy containers, duplicate elements cannot be inserted. * * To initialize a tree you have to call tommy_tree_init() specifying a comparison * function that will define the order in the tree. @@ -22,31 +21,31 @@ * To insert elements in the tree you have to call tommy_tree_insert() for * each element. * In the insertion call you have to specify the address of the node and the - * address of the element. - * The address of the element is used to initialize the tommy_node::data field + * address of the object. + * The address of the object is used to initialize the tommy_node::data field * of the node. * * \code - * struct element { + * struct object { * int value; * // other fields * tommy_tree_node node; * }; * - * struct element* elem = malloc(sizeof(struct element)); // creates the element + * struct object* obj = malloc(sizeof(struct object)); // creates the object * - * elem->value = ...; // initializes the element + * obj->value = ...; // initializes the object * - * tommy_tree_insert(&tree, &elem->node, elem); // inserts the element + * tommy_tree_insert(&tree, &obj->node, obj); // inserts the object * \endcode * - * To find the first element with a given key in tree order, call - * tommy_tree_search() providing the key to search. + * To find an element in the tree you have to call tommy_tree_search() providing + * the key to search. * * \code - * struct element value_to_find = { 1 }; - * struct element* elem = tommy_tree_search(&tree, &value_to_find); - * if (!elem) { + * struct object value_to_find = { 1 }; + * struct object* obj = tommy_tree_search(&tree, &value_to_find); + * if (!obj) { * // not found * } else { * // found @@ -60,36 +59,22 @@ * \code * tommy_tree_node* i = tommy_tree_head(&tree); * while (i) { - * struct element* elem = i->data; // gets the element pointer + * struct object* obj = i->data; // gets the object pointer * - * printf("%d\n", elem->value); // process the element + * printf("%d\n", obj->value); // process the object * * i = tommy_tree_next(i); // go to the next element * } * \endcode * - * \anchor tommy_tree_iterator_validity - * Iterators are pointers to nodes contained in the tree. - * Inserting elements or removing other nodes between iterator calls does not - * invalidate pointers to nodes that remain in the tree. - * Subsequent next/prev calls use the current contents and ordering; - * inserted elements may become visible and removed elements are skipped. - * - * Removing the pointed node invalidates its use as an iterator, even if - * its storage has not been freed. To continue traversal after removing - * the current node, obtain the next or previous node before removal. - * - * Keys and fields used for ordering must not be modified while contained. - * Concurrent traversal and modification require external synchronization. - * - * To remove the first element with a given key in tree order, call - * tommy_tree_remove() providing the key to search and remove. + * To remove an element from the tree you have to call tommy_tree_remove() + * providing the key to search and remove. * * \code - * struct element value_to_remove = { 1 }; - * struct element* elem = tommy_tree_remove(&tree, &value_to_remove); - * if (elem) { - * free(elem); // frees the element allocated memory + * struct object value_to_remove = { 1 }; + * struct object* obj = tommy_tree_remove(&tree, &value_to_remove); + * if (obj) { + * free(obj); // frees the object allocated memory * } * \endcode * @@ -97,18 +82,18 @@ * address using tommy_tree_remove_existing(). * * \code - * struct element* elem = ...; - * tommy_tree_remove_existing(&tree, &elem->node); - * free(elem); + * struct object* obj = ...; + * tommy_tree_remove_existing(&tree, &obj->node); + * free(obj); * \endcode * * To remove the smallest or greatest element you can use tommy_tree_remove_head() * or tommy_tree_remove_tail(). * * \code - * struct element* elem = tommy_tree_remove_head(&tree); - * if (elem) { - * free(elem); + * struct object* obj = tommy_tree_remove_head(&tree); + * if (obj) { + * free(obj); * } * \endcode * @@ -117,23 +102,22 @@ * This can be done with the tommy_tree_foreach() function. * * \code - * // deallocates all the elements iterating the tree + * // deallocates all the objects iterating the tree * tommy_tree_foreach(&tree, free); * \endcode */ -#ifndef TOMMY_TREE_H -#define TOMMY_TREE_H +#ifndef __TOMMYTREE_H +#define __TOMMYTREE_H #include "tommytypes.h" -#include "tommylist.h" /******************************************************************************/ /* tree */ /** * Tree node. - * This is the node that you have to include inside your elements. + * This is the node that you have to include inside your objects. * The node must preserve its natural alignment because the tree stores the * balance factor in the two least significant bits of the parent pointer. */ @@ -151,7 +135,6 @@ typedef struct tommy_tree_struct { /** * Initializes the tree. - * \param tree The tree. * \param cmp The comparison function that defines the order in the tree. */ tommy_inline void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) @@ -162,81 +145,40 @@ tommy_inline void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) } /** - * Exchanges the contents of two initialized trees, including their comparison functions. - * Each comparison function moves with its root to preserve the tree order. - * The trees must not share nodes. Nodes are not accessed or modified. - * Passing the same tree twice has no effect. - * \param first The first tree. - * \param second The second tree. - * \note This operation is O(1). + * Inserts an element in the tree. + * If the element is already present, it's not inserted again. + * Check the return value to identify if the element was already present or not. + * You have to provide the pointer of the node embedded into the object and + * the pointer to the object. + * \param node Pointer to the node embedded into the object to insert. + * \param data Pointer to the object to insert. + * \return The element in the tree. Either the already existing one, or the one just inserted. */ -tommy_inline void tommy_tree_swap(tommy_tree* first, tommy_tree* second) -{ - tommy_tree tmp = *first; - *first = *second; - *second = tmp; -} - -/** - * Removes all elements, preserving the comparison function. - * The tree remains initialized and can be reused immediately. - * Elements are not freed and nodes are not accessed or modified. - * Their links must not be used to traverse the previous contents. - * You can call this function after tommy_tree_foreach() has freed the elements. - * \param tree The tree. - * \note This operation is O(1). - */ -tommy_inline void tommy_tree_clear(tommy_tree* tree) -{ - tree->root = 0; - tree->count = 0; -} - -/** - * Inserts an element in the tree, allowing duplicate keys. - * Equal elements are visited in insertion order. - * You have to provide the pointer of the node embedded into the element and - * the pointer to the element. - * \param tree The tree. - * \param node Pointer to the node embedded into the element to insert. - * \param data Pointer to the element to insert. - */ -TOMMY_API void tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); - -/** - * Inserts an element only if no equal element is already contained. - * If found, the first equal element in tree order is returned and the - * candidate node is left unchanged. Otherwise the candidate is inserted - * and data is returned. - * \param tree The tree. - * \param node The candidate node. It must not belong to any container. - * \param data Pointer to the element to insert. - * \return The first equal element, or data if the candidate was inserted. - */ -TOMMY_API void* tommy_tree_insert_unique(tommy_tree* tree, tommy_tree_node* node, void* data); - -/** - * Transfers all elements from the tree to the tail of a list in tree order. - * Existing list elements remain before the transferred elements. The tree is - * left empty and initialized with its original comparison function. - * The list must be initialized and must not share nodes with the tree. - * Elements are not freed, and the node data and index fields are left unchanged. - * The index field no longer represents a valid tree parent after the transfer. - * \param tree The tree to drain. - * \param list The destination list. - * \note This operation is O(n), using O(log n) stack space. - */ -TOMMY_API void tommy_tree_to_list(tommy_tree* tree, tommy_list* list); +TOMMY_API void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); /** * Removes an element from the tree. - * You must already have the address of the node to remove. - * \param tree The tree. - * \param node The node to remove. - * \return The removed element. + * You must already have the address of the element to remove. + * \return The tommy_node::data field of the node removed. */ TOMMY_API void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node); + +/** \internal + * Calls the specified function for each element in the tree. + * \param root Root node of the tree. + * \param func Function to call. + */ +TOMMY_API void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func); + +/** \internal + * Calls the specified function with an argument for each element in the tree. + * \param root Root node of the tree. + * \param func Function to call. + * \param arg Custom argument for the function. + */ +TOMMY_API void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg); + /** * Gets the parent of the specified node. * \param node Node contained in the tree. @@ -249,13 +191,13 @@ tommy_inline tommy_tree_node* tommy_tree_parent(tommy_tree_node* node) /** * Gets the node following the specified one in tree order. - * See \ref tommy_tree_iterator_validity for iterator validity across insertions and removals. * \param node Node contained in the tree. - * \return The following node, or 0 for the tail node. - * \note This operation is O(log n), and amortized O(1) during full traversal. + * \return The following node. For the tail node 0 is returned. */ tommy_inline tommy_tree_node* tommy_tree_next(tommy_tree_node* node) { + tommy_tree_node* parent; + if (node->next) { node = node->next; while (node->prev) @@ -263,7 +205,7 @@ tommy_inline tommy_tree_node* tommy_tree_next(tommy_tree_node* node) return node; } - tommy_tree_node* parent = tommy_tree_parent(node); + parent = tommy_tree_parent(node); while (parent && node == parent->next) { node = parent; parent = tommy_tree_parent(parent); @@ -274,13 +216,13 @@ tommy_inline tommy_tree_node* tommy_tree_next(tommy_tree_node* node) /** * Gets the node preceding the specified one in tree order. - * See \ref tommy_tree_iterator_validity for iterator validity across insertions and removals. * \param node Node contained in the tree. - * \return The preceding node, or 0 for the head node. - * \note This operation is O(log n), and amortized O(1) during full traversal. + * \return The preceding node. For the head node 0 is returned. */ tommy_inline tommy_tree_node* tommy_tree_prev(tommy_tree_node* node) { + tommy_tree_node* parent; + if (node->prev) { node = node->prev; while (node->next) @@ -288,7 +230,7 @@ tommy_inline tommy_tree_node* tommy_tree_prev(tommy_tree_node* node) return node; } - tommy_tree_node* parent = tommy_tree_parent(node); + parent = tommy_tree_parent(node); while (parent && node == parent->prev) { node = parent; parent = tommy_tree_parent(parent); @@ -298,10 +240,8 @@ tommy_inline tommy_tree_node* tommy_tree_prev(tommy_tree_node* node) } /** - * Gets the head (smallest) node in the tree. - * \param tree The tree. - * \return The head node, or 0 if the tree is empty. - * \note This operation is O(log n). + * Gets the head (smallest) element in the tree. + * \return The head node. For empty trees 0 is returned. */ tommy_inline tommy_tree_node* tommy_tree_head(tommy_tree* tree) { @@ -317,10 +257,8 @@ tommy_inline tommy_tree_node* tommy_tree_head(tommy_tree* tree) } /** - * Gets the tail (greatest) node in the tree. - * \param tree The tree. - * \return The tail node, or 0 if the tree is empty. - * \note This operation is O(log n). + * Gets the tail (greatest) element in the tree. + * \return The tail node. For empty trees 0 is returned. */ tommy_inline tommy_tree_node* tommy_tree_tail(tommy_tree* tree) { @@ -338,8 +276,7 @@ tommy_inline tommy_tree_node* tommy_tree_tail(tommy_tree* tree) /** * Removes and returns the head (smallest) element. * If the tree is empty, 0 is returned. - * \param tree The tree. - * \return The removed element, or 0 if the tree is empty. + * \return The removed element, or 0 if empty. */ tommy_inline void* tommy_tree_remove_head(tommy_tree* tree) { @@ -354,8 +291,7 @@ tommy_inline void* tommy_tree_remove_head(tommy_tree* tree) /** * Removes and returns the tail (greatest) element. * If the tree is empty, 0 is returned. - * \param tree The tree. - * \return The removed element, or 0 if the tree is empty. + * \return The removed element, or 0 if empty. */ tommy_inline void* tommy_tree_remove_tail(tommy_tree* tree) { @@ -367,53 +303,44 @@ tommy_inline void* tommy_tree_remove_tail(tommy_tree* tree) return tommy_tree_remove_existing(tree, node); } -/** - * Finds the first equal node in the tree with a specific comparison function. - * - * Like tommy_tree_find() but you can specify a different comparison function. - * Note that this function must define a suborder of the original one. - * The first equal node in tree order is returned. - * - * The cmp_arg argument will be the first argument of the comparison function, - * and it can be of a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first equal node in tree order, or 0 if none. +/** \internal + * Searches a node in the tree. + * If the element is not found, 0 is returned. + * \param cmp Comparison function. + * \param node Node to start the search from. + * \param data Element used for comparison. + * \return The first node found, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_find_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +tommy_inline tommy_tree_node* tommy_tree_search_node(tommy_compare_func* cmp, tommy_tree_node* node, void* data) { - tommy_tree_node* node = tree->root; - tommy_tree_node* candidate = 0; - while (node) { - /* comparators must not change the tree, so retain the data across the call */ - void* data = node->data; - int c = cmp(cmp_arg, data); + int c = cmp(data, node->data); if (c < 0) node = node->prev; else if (c > 0) node = node->next; - else { - candidate = node; - node = node->prev; - } + else + return node; } - return candidate; + return 0; } /** - * Finds the first equal node in the tree. + * Searches an element in the tree. * If the element is not found, 0 is returned. - * \param tree The tree. * \param data Element used for comparison. - * \return The first equal node in tree order, or 0 if none. + * \return The first element found, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_find(tommy_tree* tree, const void* data) +tommy_inline void* tommy_tree_search(tommy_tree* tree, void* data) { - return tommy_tree_find_compare(tree, tree->cmp, data); + tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); + + if (!node) + return 0; + + return node->data; } /** @@ -421,65 +348,42 @@ tommy_inline tommy_tree_node* tommy_tree_find(tommy_tree* tree, const void* data * * Like tommy_tree_search() but you can specify a different comparison function. * Note that this function must define a suborder of the original one. - * The first equal element in tree order is returned. * - * The cmp_arg argument will be the first argument of the comparison function, - * and it can be of a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first equal element in tree order, or 0 if none. + * The ::data argument will be the first argument of the comparison function, + * and it can be of a different type than the objects in the tree. */ -tommy_inline void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +tommy_inline void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) { - tommy_tree_node* node = tommy_tree_find_compare(tree, cmp, cmp_arg); + tommy_tree_node* node = tommy_tree_search_node(cmp, tree->root, cmp_arg); - return node ? node->data : 0; + if (!node) + return 0; + + return node->data; } -/** - * Searches an element in the tree. - * If the element is not found, 0 is returned. - * \param tree The tree. +/** \internal + * Searches a node in the tree with key greater or equal than the specified one. + * If no such element exists, 0 is returned. + * \param cmp Comparison function. + * \param node Node to start the search from. * \param data Element used for comparison. - * \return The first equal element in tree order, or 0 if none. + * \return The first node found, or 0 if none. */ -tommy_inline void* tommy_tree_search(tommy_tree* tree, const void* data) +tommy_inline tommy_tree_node* tommy_tree_search_greater_equal_node(tommy_compare_func* cmp, tommy_tree_node* node, void* data) { - tommy_tree_node* node = tommy_tree_find(tree, data); - - return node ? node->data : 0; -} - -/** - * Finds a node in the tree with key greater or equal than the specified one with a specific comparison function. - * - * Like tommy_tree_find_greater_equal() but you can specify a different comparison function. - * Note that this function must define a suborder of the original one. - * The first qualifying node in tree order is returned. - * - * The cmp_arg argument will be the first argument of the comparison function, - * and it can be of a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first node in tree order with key greater or equal, or 0 if none. - */ -tommy_inline tommy_tree_node* tommy_tree_find_greater_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* node = tree->root; tommy_tree_node* candidate = 0; while (node) { - /* comparators must not change the tree, so retain the data across the call */ - void* data = node->data; - int c = cmp(cmp_arg, data); + int c = cmp(data, node->data); - if (c <= 0) { + if (c < 0) { candidate = node; node = node->prev; - } else { + } else if (c > 0) { node = node->next; + } else { + return node; } } @@ -487,15 +391,19 @@ tommy_inline tommy_tree_node* tommy_tree_find_greater_equal_compare(tommy_tree* } /** - * Finds a node in the tree with key greater or equal than the specified one. + * Searches an element in the tree with key greater or equal than the specified one. * If no such element exists, 0 is returned. - * \param tree The tree. * \param data Element used for comparison. - * \return The first node in tree order with key greater or equal, or 0 if none. + * \return The first element found, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_find_greater_equal(tommy_tree* tree, const void* data) +tommy_inline void* tommy_tree_search_greater_equal(tommy_tree* tree, void* data) { - return tommy_tree_find_greater_equal_compare(tree, tree->cmp, data); + tommy_tree_node* node = tommy_tree_search_greater_equal_node(tree->cmp, tree->root, data); + + if (!node) + return 0; + + return node->data; } /** @@ -503,60 +411,42 @@ tommy_inline tommy_tree_node* tommy_tree_find_greater_equal(tommy_tree* tree, co * * Like tommy_tree_search_greater_equal() but you can specify a different comparison function. * Note that this function must define a suborder of the original one. - * The first qualifying element in tree order is returned. * - * The cmp_arg argument will be the first argument of the comparison function, - * and it can be of a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first element in tree order with key greater or equal, or 0 if none. + * The ::data argument will be the first argument of the comparison function, + * and it can be of a different type than the objects in the tree. */ -tommy_inline void* tommy_tree_search_greater_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +tommy_inline void* tommy_tree_search_greater_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) { - tommy_tree_node* node = tommy_tree_find_greater_equal_compare(tree, cmp, cmp_arg); + tommy_tree_node* node = tommy_tree_search_greater_equal_node(cmp, tree->root, cmp_arg); - return node ? node->data : 0; + if (!node) + return 0; + + return node->data; } -/** - * Searches an element in the tree with key greater or equal than the specified one. +/** \internal + * Searches a node in the tree with key less or equal than the specified one. * If no such element exists, 0 is returned. - * \param tree The tree. + * \param cmp Comparison function. + * \param node Node to start the search from. * \param data Element used for comparison. - * \return The first element in tree order with key greater or equal, or 0 if none. + * \return The first node found, or 0 if none. */ -tommy_inline void* tommy_tree_search_greater_equal(tommy_tree* tree, const void* data) +tommy_inline tommy_tree_node* tommy_tree_search_less_equal_node(tommy_compare_func* cmp, tommy_tree_node* node, void* data) { - tommy_tree_node* node = tommy_tree_find_greater_equal(tree, data); - - return node ? node->data : 0; -} - -/** - * Finds the first node in tree order with key strictly greater than the specified one using a specific comparison function. - * The function must define a suborder of the tree comparison function. - * Equivalent elements are excluded from the result. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first strictly greater node, or 0 if none. - */ -tommy_inline tommy_tree_node* tommy_tree_find_greater_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* node = tree->root; tommy_tree_node* candidate = 0; while (node) { - /* comparators must not change the tree, so retain the data across the call */ - void* data = node->data; - int c = cmp(cmp_arg, data); + int c = cmp(data, node->data); if (c < 0) { - candidate = node; node = node->prev; - } else { + } else if (c > 0) { + candidate = node; node = node->next; + } else { + return node; } } @@ -564,92 +454,19 @@ tommy_inline tommy_tree_node* tommy_tree_find_greater_compare(tommy_tree* tree, } /** - * Finds the first node in tree order with key strictly greater than the specified one. + * Searches an element in the tree with key less or equal than the specified one. * If no such element exists, 0 is returned. - * \param tree The tree. * \param data Element used for comparison. - * \return The first strictly greater node, or 0 if none. + * \return The first element found, or 0 if none. */ -tommy_inline tommy_tree_node* tommy_tree_find_greater(tommy_tree* tree, const void* data) +tommy_inline void* tommy_tree_search_less_equal(tommy_tree* tree, void* data) { - return tommy_tree_find_greater_compare(tree, tree->cmp, data); -} + tommy_tree_node* node = tommy_tree_search_less_equal_node(tree->cmp, tree->root, data); -/** - * Searches the first element in tree order with key strictly greater than the specified one using a specific comparison function. - * The function must define a suborder of the tree comparison function. - * Equivalent elements are excluded from the result. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first strictly greater element, or 0 if none. - */ -tommy_inline void* tommy_tree_search_greater_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* node = tommy_tree_find_greater_compare(tree, cmp, cmp_arg); + if (!node) + return 0; - return node ? node->data : 0; -} - -/** - * Searches the first element in tree order with key strictly greater than the specified one. - * If no such element exists, 0 is returned. - * \param tree The tree. - * \param data Element used for comparison. - * \return The first strictly greater element, or 0 if none. - */ -tommy_inline void* tommy_tree_search_greater(tommy_tree* tree, const void* data) -{ - tommy_tree_node* node = tommy_tree_find_greater(tree, data); - - return node ? node->data : 0; -} - -/** - * Finds a node in the tree with key less or equal than the specified one with a specific comparison function. - * - * Like tommy_tree_find_less_equal() but you can specify a different comparison function. - * Note that this function must define a suborder of the original one. - * The last qualifying node in tree order is returned. - * - * The cmp_arg argument will be the first argument of the comparison function, - * and it can be of a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The last node in tree order with key less or equal, or 0 if none. - */ -tommy_inline tommy_tree_node* tommy_tree_find_less_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* node = tree->root; - tommy_tree_node* candidate = 0; - - while (node) { - /* comparators must not change the tree, so retain the data across the call */ - void* data = node->data; - int c = cmp(cmp_arg, data); - - if (c < 0) { - node = node->prev; - } else { - candidate = node; - node = node->next; - } - } - - return candidate; -} - -/** - * Finds a node in the tree with key less or equal than the specified one. - * If no such element exists, 0 is returned. - * \param tree The tree. - * \param data Element used for comparison. - * \return The last node in tree order with key less or equal, or 0 if none. - */ -tommy_inline tommy_tree_node* tommy_tree_find_less_equal(tommy_tree* tree, const void* data) -{ - return tommy_tree_find_less_equal_compare(tree, tree->cmp, data); + return node->data; } /** @@ -657,138 +474,34 @@ tommy_inline tommy_tree_node* tommy_tree_find_less_equal(tommy_tree* tree, const * * Like tommy_tree_search_less_equal() but you can specify a different comparison function. * Note that this function must define a suborder of the original one. - * The last qualifying element in tree order is returned. * - * The cmp_arg argument will be the first argument of the comparison function, - * and it can be of a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The last element in tree order with key less or equal, or 0 if none. + * The ::data argument will be the first argument of the comparison function, + * and it can be of a different type than the objects in the tree. */ -tommy_inline void* tommy_tree_search_less_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) +tommy_inline void* tommy_tree_search_less_equal_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) { - tommy_tree_node* node = tommy_tree_find_less_equal_compare(tree, cmp, cmp_arg); + tommy_tree_node* node = tommy_tree_search_less_equal_node(cmp, tree->root, cmp_arg); - return node ? node->data : 0; -} - -/** - * Searches an element in the tree with key less or equal than the specified one. - * If no such element exists, 0 is returned. - * \param tree The tree. - * \param data Element used for comparison. - * \return The last element in tree order with key less or equal, or 0 if none. - */ -tommy_inline void* tommy_tree_search_less_equal(tommy_tree* tree, const void* data) -{ - tommy_tree_node* node = tommy_tree_find_less_equal(tree, data); - - return node ? node->data : 0; -} - -/** - * Finds the last node in tree order with key strictly less than the specified one using a specific comparison function. - * The function must define a suborder of the tree comparison function. - * Equivalent elements are excluded from the result. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The last strictly less node, or 0 if none. - */ -tommy_inline tommy_tree_node* tommy_tree_find_less_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* node = tree->root; - tommy_tree_node* candidate = 0; - - while (node) { - /* comparators must not change the tree, so retain the data across the call */ - void* data = node->data; - int c = cmp(cmp_arg, data); - - if (c > 0) { - candidate = node; - node = node->next; - } else { - node = node->prev; - } - } - - return candidate; -} - -/** - * Finds the last node in tree order with key strictly less than the specified one. - * If no such element exists, 0 is returned. - * \param tree The tree. - * \param data Element used for comparison. - * \return The last strictly less node, or 0 if none. - */ -tommy_inline tommy_tree_node* tommy_tree_find_less(tommy_tree* tree, const void* data) -{ - return tommy_tree_find_less_compare(tree, tree->cmp, data); -} - -/** - * Searches the last element in tree order with key strictly less than the specified one using a specific comparison function. - * The function must define a suborder of the tree comparison function. - * Equivalent elements are excluded from the result. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The last strictly less element, or 0 if none. - */ -tommy_inline void* tommy_tree_search_less_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* node = tommy_tree_find_less_compare(tree, cmp, cmp_arg); - - return node ? node->data : 0; -} - -/** - * Searches the last element in tree order with key strictly less than the specified one. - * If no such element exists, 0 is returned. - * \param tree The tree. - * \param data Element used for comparison. - * \return The last strictly less element, or 0 if none. - */ -tommy_inline void* tommy_tree_search_less(tommy_tree* tree, const void* data) -{ - tommy_tree_node* node = tommy_tree_find_less(tree, data); - - return node ? node->data : 0; -} - -/** - * Searches and removes an element from the tree with a specific comparison function. - * The comparison function must define a suborder of the original one. - * The cmp_arg argument is passed as the first argument of the comparison - * function and may have a different type than the elements in the tree. - * \param tree The tree. - * \param cmp Comparison function called with cmp_arg and the element in the tree. - * \param cmp_arg Search key, which may have a different type from tree elements. - * \return The first equal element in tree order, or 0 if not found. - */ -tommy_inline void* tommy_tree_remove_compare(tommy_tree* tree, tommy_compare_func* cmp, const void* cmp_arg) -{ - tommy_tree_node* candidate = tommy_tree_find_compare(tree, cmp, cmp_arg); - - if (!candidate) + if (!node) return 0; - return tommy_tree_remove_existing(tree, candidate); + return node->data; } /** - * Searches and removes an element from the tree. + * Searches and removes an element. * If the element is not found, 0 is returned. - * \param tree The tree. * \param data Element used for comparison. - * \return The first equal element in tree order, or 0 if not found. + * \return The removed element, or 0 if not found. */ -tommy_inline void* tommy_tree_remove(tommy_tree* tree, const void* data) +tommy_inline void* tommy_tree_remove(tommy_tree* tree, void* data) { - return tommy_tree_remove_compare(tree, tree->cmp, data); + tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); + + if (!node) + return 0; + + return tommy_tree_remove_existing(tree, node); } /** @@ -807,58 +520,46 @@ tommy_inline void* tommy_tree_remove(tommy_tree* tree, const void* data) * * ... * - * // creates an element - * struct element* elem = malloc(sizeof(struct element)); + * // creates an object + * struct object* obj = malloc(sizeof(struct object)); * * ... * * // insert it in the tree - * tommy_tree_insert(&tree, &elem->node, elem); + * tommy_tree_insert(&tree, &obj->node, obj); * * ... * - * // deallocates all the elements iterating the tree + * // deallocates all the objects iterating the tree * tommy_tree_foreach(&tree, free); - * tommy_tree_clear(&tree); // resets the tree before reuse * \endcode - * - * If the callback frees the elements, the tree retains dangling node - * pointers. Call tommy_tree_clear() or tommy_tree_init() before reusing it. - * The tree has no storage to release. - * \param tree The tree. - * \param func Function to call with each element. */ -TOMMY_API void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func); +tommy_inline void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) +{ + tommy_tree_foreach_node(tree->root, func); +} /** * Calls the specified function with an argument for each element in the tree. - * The iteration order and callback rules are the same as tommy_tree_foreach(). - * The callback may deallocate the current element. - * Adding or removing elements from inside the callback is not allowed. - * \param tree The tree. - * \param func Function to call with each element. - * \param arg Argument to pass to the function. */ -TOMMY_API void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg); +tommy_inline void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) +{ + tommy_tree_foreach_arg_node(tree->root, func, arg); +} /** - * Checks whether the tree is empty. - * \param tree The tree. - * \return Nonzero if the tree is empty, 0 otherwise. - * \note This operation is O(1). + * Checks if empty. + * \return If the tree is empty. */ -tommy_inline tommy_bool_t tommy_tree_empty(const tommy_tree* tree) +tommy_inline tommy_bool_t tommy_tree_empty(tommy_tree* tree) { return tree->root == 0; } /** * Gets the number of elements. - * \param tree The tree. - * \return The number of elements in the tree. - * \note This operation is O(1). */ -tommy_inline tommy_size_t tommy_tree_count(const tommy_tree* tree) +tommy_inline tommy_size_t tommy_tree_count(tommy_tree* tree) { return tree->count; } @@ -866,10 +567,8 @@ tommy_inline tommy_size_t tommy_tree_count(const tommy_tree* tree) /** * Gets the size of allocated memory. * It includes the size of the ::tommy_tree_node of the stored elements. - * \param tree The tree. - * \return Total memory allocated by the tree, in bytes. */ -TOMMY_API tommy_size_t tommy_tree_memory_usage(const tommy_tree* tree); +TOMMY_API tommy_size_t tommy_tree_memory_usage(tommy_tree* tree); #endif diff --git a/tommyds/tommytypes.h b/tommyds/tommytypes.h index d2dd85ec..a63894a7 100644 --- a/tommyds/tommytypes.h +++ b/tommyds/tommytypes.h @@ -5,8 +5,8 @@ * Generic types. */ -#ifndef TOMMY_TYPES_H -#define TOMMY_TYPES_H +#ifndef __TOMMYTYPES_H +#define __TOMMYTYPES_H /******************************************************************************/ /* types */ @@ -14,18 +14,15 @@ #include #ifdef _MSC_VER -#include typedef unsigned tommy_uint32_t; /**< Generic uint32_t type. */ typedef unsigned _int64 tommy_uint64_t; /**< Generic uint64_t type. */ typedef size_t tommy_uintptr_t; /**< Generic uintptr_t type. */ #ifdef _WIN64 #define TOMMY_SIZE_BIT 64 -#define TOMMY_SIZE_MAX _UI64_MAX typedef unsigned _int64 tommy_size_t; /**< Generic size_t type. */ typedef _int64 tommy_ssize_t; /**< Generic ssize_t type. */ #else #define TOMMY_SIZE_BIT 32 -#define TOMMY_SIZE_MAX UINT_MAX typedef unsigned tommy_size_t; /**< Generic size_t type. */ typedef int tommy_ssize_t; /**< Generic ssize_t type. */ #endif @@ -36,12 +33,10 @@ typedef uint64_t tommy_uint64_t; /**< Generic uint64_t type. */ typedef uintptr_t tommy_uintptr_t; /**< Generic uintptr_t type. */ #if SIZE_MAX == UINT64_MAX #define TOMMY_SIZE_BIT 64 -#define TOMMY_SIZE_MAX SIZE_MAX typedef uint64_t tommy_size_t; /**< Generic size_t type. */ typedef int64_t tommy_ssize_t; /**< Generic ssize_t type. */ #elif SIZE_MAX == UINT32_MAX #define TOMMY_SIZE_BIT 32 -#define TOMMY_SIZE_MAX SIZE_MAX typedef uint32_t tommy_size_t; /**< Generic size_t type. */ typedef int32_t tommy_ssize_t; /**< Generic ssize_t type. */ #else @@ -56,7 +51,7 @@ typedef int tommy_bool_t; /**< Generic boolean type. */ * Generic unsigned integer type. * * It has no specific size, as it is used to store only small values. - * To make the code more efficient, a full 32-bit integer is used. + * To make the code more efficient, a full 32 bit integer is used. */ typedef tommy_uint32_t tommy_uint_t; @@ -85,57 +80,7 @@ typedef tommy_uint32_t tommy_uint_t; /** * Generic malloc(), calloc(), realloc() and free() functions. - * - * Redefine them to what you need. By default they map to the C malloc(), - * calloc(), realloc() and free(). - * - * \par Infallible Allocation Invariant - * Tommy assumes that dynamic memory allocation never fails and that these - * functions always return a valid, non-zero pointer. The library does not - * check for allocation failures and does not return error codes for - * out-of-memory (OOM) conditions. - * - * \par Rationale - * This choice reflects the memory model of modern general-purpose operating - * systems: - * - **Virtual Memory Overcommit:** On modern systems (such as Linux with default - * overcommit heuristics), malloc() reserves virtual address space and almost - * never returns 0 on 64-bit systems. Physical memory is committed on first write, - * meaning actual exhaustion triggers the kernel OOM Killer (SIGKILL) rather - * than an allocation failure. Checking for null returns in userspace often - * provides a false sense of security. - * - **Swap Thrashing:** In systems backed by swap, severe memory pressure causes - * destructive disk thrashing, degrading throughput by orders of magnitude and - * rendering the process unusable long before an allocation call fails. - * - * \par Advantages - * - **Zero Error-Path Overhead:** No branch predictions, conditional jumps, or - * status checks inside critical insertion and resizing loops, maximizing CPU - * cache efficiency and execution speed. - * - **Clean and Lean API:** Functions (such as insertion and resizing) can remain - * simple (often returning void or direct element pointers) without cluttering - * client code with error-propagation boilerplate. - * - **Deterministic Consistency:** Avoids complex transactional rollback logic - * otherwise required if an allocation failed halfway through dynamic table - * growth or node migration. - * - **Industry Alignment:** Follows the established design of high-performance - * systems libraries (e.g., GLib's g_malloc, Git's xmalloc, Redis's zmalloc, - * and default C++ operator new). - * - * \par Trade-offs and Limitations - * - **Embedded and Bare-Metal Systems:** Systems without an MMU/virtual memory - * (e.g., RTOS, microcontrollers) or strict overcommit disabled (vm.overcommit_memory=2) - * do return 0 on heap exhaustion. - * - **Multi-Tenant Services:** Infrastructure daemons (e.g., database engines) that - * must isolate a single client's excessive memory request without crashing the - * entire process cannot rely on unchecked allocations. - * - * \par Custom Interception - * If your environment requires custom handling (such as memory tracking, structured - * aborts, or logging), redefine tommy_malloc, tommy_calloc, tommy_realloc, and - * tommy_free before including Tommy headers (or via compiler flags). - * Custom allocators must terminate the process (e.g., abort()) or initiate an - * unwinding non-local exit (e.g., longjmp or C++ exception) rather than returning 0. + * Redefine them to what you need. By default they map to the C malloc(), calloc(), realloc() and free(). */ #if !defined(tommy_malloc) || !defined(tommy_calloc) || !defined(tommy_realloc) || !defined(tommy_free) #include @@ -154,7 +99,7 @@ typedef tommy_uint32_t tommy_uint_t; #endif /******************************************************************************/ -/* modifiers */ +/* modificators */ /** \internal * Definition of TOMMY_API. @@ -179,7 +124,7 @@ typedef tommy_uint32_t tommy_uint_t; * Definition of the restrict keyword if available. */ #if !defined(tommy_restrict) -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 199901L +#if __STDC_VERSION__ >= 199901L #define tommy_restrict restrict #elif defined(_MSC_VER) || defined(__GNUC__) #define tommy_restrict __restrict @@ -214,12 +159,12 @@ typedef tommy_uint32_t tommy_uint_t; /* key/hash */ /** - * Type used in indexed data structures to store the key of an element. + * Type used in indexed data structures to store the key of an object. */ typedef tommy_size_t tommy_key_t; /** - * Type used in hashtables to store the hash of an element. + * Type used in hashtables to store the hash of an object. */ typedef tommy_size_t tommy_hash_t; @@ -229,11 +174,11 @@ typedef tommy_size_t tommy_hash_t; /** * Data structure node. * This node type is shared between all the data structures and used to store some - * info directly into the elements you want to store. + * info directly into the objects you want to store. * * A typical declaration is: * \code - * struct element { + * struct object { * tommy_node node; * // other fields * }; @@ -242,7 +187,7 @@ typedef tommy_size_t tommy_hash_t; typedef struct tommy_node_struct { /** * Next node. - * The tail node has it at 0, like a 0-terminated list. + * The tail node has it at 0, like a 0 terminated list. */ struct tommy_node_struct* next; @@ -253,9 +198,8 @@ typedef struct tommy_node_struct { struct tommy_node_struct* prev; /** - * Pointer to the element containing the node. + * Pointer to the object containing the node. * This field is initialized when inserting nodes into a data structure. - * The element pointer passed to insertion functions must not be 0. */ void* data; @@ -263,7 +207,6 @@ typedef struct tommy_node_struct { * Index of the node. * With tries this field is used to store the key. * With hashtables this field is used to store the hash value. - * With trees this field is used to store the parent pointer and balance factor. * With lists this field is not used. */ tommy_size_t index; @@ -274,20 +217,23 @@ typedef struct tommy_node_struct { /** * Compare function for elements. + * \param obj_a Pointer to the first object to compare. + * \param obj_b Pointer to the second object to compare. + * \return <0 if the first element is less than the second, ==0 if equal, >0 if greater. * * This function is like the C strcmp(). * * \code - * struct element { + * struct object { * tommy_node node; * int value; * }; * - * int compare(const void* elem_a, const void* elem_b) + * int compare(const void* obj_a, const void* obj_b) * { - * if (((const struct element*)elem_a)->value < ((const struct element*)elem_b)->value) + * if (((const struct object*)obj_a)->value < ((const struct object*)obj_b)->value) * return -1; - * if (((const struct element*)elem_a)->value > ((const struct element*)elem_b)->value) + * if (((const struct object*)obj_a)->value > ((const struct object*)obj_b)->value) * return 1; * return 0; * } @@ -295,62 +241,64 @@ typedef struct tommy_node_struct { * tommy_list_sort(&list, compare); * \endcode * - * \param elem_a Pointer to the first element to compare. - * \param elem_b Pointer to the second element to compare. - * \return <0 if the first element is less than the second, ==0 if equal, >0 if greater. */ -typedef int tommy_compare_func(const void* elem_a, const void* elem_b); +typedef int tommy_compare_func(const void* obj_a, const void* obj_b); /** * Search function for elements. + * \param arg Pointer to the value to search as passed at the search function. + * \param obj Pointer to the object to compare to. + * \return ==0 if the value matches the element. !=0 if different. + * + * The first argument is a pointer to the value to search exactly + * as it's passed at the search function called. + * The second argument is a pointer to the object inside the hashtable to compare. + * + * The return value has to be 0 if the values are equal. != 0 if they are different. * * \code - * struct element { + * struct object { * tommy_node node; * int value; * }; * - * int compare(const void* arg, const void* elem) + * int compare(const void* arg, const void* obj) * { * const int* value_to_find = arg; - * const struct element* element_to_compare = elem; + * const struct object* object_to_compare = obj; * - * return *value_to_find != element_to_compare->value; + * return *value_to_find != object_to_compare->value; * } * * int value_to_find = 1; - * struct element* elem = tommy_hashtable_search(&hashtable, compare, &value_to_find, tommy_inthash_u32(value_to_find)); - * if (!elem) { + * struct object* obj = tommy_hashtable_search(&hashtable, compare, &value_to_find, tommy_inthash_u32(value_to_find)); + * if (!obj) { * // not found * } else { * // found * } * \endcode * - * \param arg Pointer to the value to search as passed to the search function. - * \param elem Pointer to the element to compare to. - * \return ==0 if the value matches the element. !=0 if different. */ -typedef int tommy_search_func(const void* arg, const void* elem); +typedef int tommy_search_func(const void* arg, const void* obj); /** * Foreach function. + * \param obj Pointer to the object to iterate. * - * A typical example is to use free() to deallocate all the elements in a list. + * A typical example is to use free() to deallocate all the objects in a list. * \code - * tommy_list_foreach(&list, free); + * tommy_list_foreach(&list, (tommy_foreach_func*)free); * \endcode - * - * \param elem Pointer to the element to iterate. */ -typedef void tommy_foreach_func(void* elem); +typedef void tommy_foreach_func(void* obj); /** * Foreach function with an argument. * \param arg Pointer to a generic argument. - * \param elem Pointer to the element to iterate. + * \param obj Pointer to the object to iterate. */ -typedef void tommy_foreach_arg_func(void* arg, void* elem); +typedef void tommy_foreach_arg_func(void* arg, void* obj); /******************************************************************************/ /* bit hacks */ @@ -367,7 +315,7 @@ typedef void tommy_foreach_arg_func(void* arg, void* elem); /** \internal * Integer log2 for constants. - * You can use it only for exact powers of 2 up to 256. + * You can use it only for exact power of 2 up to 256. */ #define TOMMY_ILOG2(value) ((value) == 256 ? 8 : (value) == 128 ? 7 : (value) == 64 ? 6 : (value) == 32 ? 5 : (value) == 16 ? 4 : (value) == 8 ? 3 : (value) == 4 ? 2 : (value) == 2 ? 1 : 0) @@ -407,9 +355,9 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value) */ return __builtin_clz(value) ^ 31; #else - /* find the log base 2 of an N-bit integer in O(lg(N)) operations with multiply and lookup */ + /* Find the log base 2 of an N-bit integer in O(lg(N)) operations with multiply and lookup */ /* from http://graphics.stanford.edu/~seander/bithacks.html */ - static const unsigned char TOMMY_DE_BRUIJN_INDEX_ILOG2[32] = { + static unsigned char TOMMY_DE_BRUIJN_INDEX_ILOG2[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; @@ -424,17 +372,13 @@ tommy_inline tommy_uint_t tommy_ilog2_u32(tommy_uint32_t value) #endif } +#if TOMMY_SIZE_BIT == 64 /** * Bit scan reverse or integer log2 for 64 bits. - * Return the bit index of the most significant 1 bit. - * - * If no bit is set, the result is undefined. - * \param value Value to scan. 0 is not allowed. - * \return The index of the most significant bit set. */ tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value) { -#if defined(_MSC_VER) && defined(_WIN64) +#if defined(_MSC_VER) unsigned long count; _BitScanReverse64(&count, value); return count; @@ -449,6 +393,7 @@ tommy_inline tommy_uint_t tommy_ilog2_u64(tommy_uint64_t value) return tommy_ilog2_u32(l); #endif } +#endif /** * Bit scan forward or trailing zero count. @@ -467,28 +412,24 @@ tommy_inline tommy_uint_t tommy_ctz_u32(tommy_uint32_t value) #elif defined(__GNUC__) return __builtin_ctz(value); #else - /* count the consecutive zero bits (trailing) on the right with multiply and lookup */ + /* Count the consecutive zero bits (trailing) on the right with multiply and lookup */ /* from http://graphics.stanford.edu/~seander/bithacks.html */ static const unsigned char TOMMY_DE_BRUIJN_INDEX_CTZ[32] = { 0, 1, 28, 2, 29, 14, 24, 3, 30, 22, 20, 15, 25, 17, 4, 8, 31, 27, 13, 23, 21, 19, 16, 7, 26, 12, 18, 6, 11, 5, 10, 9 }; - return TOMMY_DE_BRUIJN_INDEX_CTZ[(tommy_uint32_t)(((value & -value) * 0x077CB531U)) >> 27]; + return TOMMY_DE_BRUIJN_INDEX_CTZ[(tommy_uint32_t)(((value & - value) * 0x077CB531U)) >> 27]; #endif } +#if TOMMY_SIZE_BIT == 64 /** * Bit scan forward or trailing zero count for 64 bits. - * Return the bit index of the least significant 1 bit. - * - * If no bit is set, the result is undefined. - * \param value Value to scan. 0 is not allowed. - * \return The index of the least significant bit set. */ tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value) { -#if defined(_MSC_VER) && defined(_WIN64) +#if defined(_MSC_VER) unsigned long count; _BitScanForward64(&count, value); return count; @@ -503,16 +444,16 @@ tommy_inline tommy_uint_t tommy_ctz_u64(tommy_uint64_t value) return tommy_ctz_u32(h) + 32; #endif } +#endif /** * Rounds up to the next power of 2. * For the value 0, the result is undefined. - * \param value Value to round up. * \return The smallest power of 2 not less than the specified value. */ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) { - /* round up to the next highest power of 2 */ + /* Round up to the next highest power of 2 */ /* from http://graphics.stanford.edu/~seander/bithacks.html */ --value; @@ -528,9 +469,6 @@ tommy_inline tommy_uint32_t tommy_roundup_pow2_u32(tommy_uint32_t value) /** * Rounds up to the next power of 2 for 64 bits. - * For the value 0, the result is undefined. - * \param value Value to round up. - * \return The smallest power of 2 not less than the specified value. */ tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value) { @@ -547,8 +485,7 @@ tommy_inline tommy_uint64_t tommy_roundup_pow2_u64(tommy_uint64_t value) } /** - * Checks if the specified word has a byte at 0. - * \param value Value to check. + * Check if the specified word has a byte at 0. * \return 0 or 1. */ tommy_inline int tommy_haszero_u32(tommy_uint32_t value) @@ -556,17 +493,7 @@ tommy_inline int tommy_haszero_u32(tommy_uint32_t value) return ((value - 0x01010101) & ~value & 0x80808080) != 0; } -/** - * Checks if the specified 64-bit word has a byte at 0. - * \param value Value to check. - * \return 0 or 1. - */ -tommy_inline int tommy_haszero_u64(tommy_uint64_t value) -{ - return ((value - 0x0101010101010101ULL) & ~value & 0x8080808080808080ULL) != 0; -} - -/** +/* * Bit depth mapping. */ #if TOMMY_SIZE_BIT == 64 @@ -580,4 +507,3 @@ tommy_inline int tommy_haszero_u64(tommy_uint64_t value) #endif #endif -