diff --git a/cmdline/state.c b/cmdline/state.c index b3fc0e07..640e2f12 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -609,6 +609,37 @@ void state_config(struct snapraid_state* state, const char* path, const char* co /* LCOV_EXCL_STOP */ } state->block_size *= KIBI; + } else if (strcmp(tag, "hash_size") == 0) { + uint32_t hash_size; + + ret = sgetu32(f, &hash_size); + if (ret < 0) { + /* LCOV_EXCL_START */ + log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (hash_size < 2) { + /* LCOV_EXCL_START */ + log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (hash_size > HASH_MAX) { + /* LCOV_EXCL_START */ + log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + /* check if it's a power of 2 */ + if ((hash_size & (hash_size - 1)) != 0) { + /* LCOV_EXCL_START */ + log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + BLOCK_HASH_SIZE = hash_size; } else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) { char device[PATH_MAX]; char* slash; @@ -1710,9 +1741,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S * - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support. * The previous 'm' entry is now deprecated, but supported for importing. * Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'. + * - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size. */ if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0 - && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0) { + && memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0 + && memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0 + ) { /* LCOV_EXCL_START */ if (memcmp(buffer, "SNAPCNT", 7) != 0) { decoding_error(path, f); @@ -2365,9 +2399,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_STOP */ } } else if (c == 'z') { - block_off_t blksize; + uint32_t block_size; - ret = sgetb32(f, &blksize); + ret = sgetb32(f, &block_size); if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); @@ -2375,7 +2409,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* LCOV_EXCL_STOP */ } - if (blksize == 0) { + if (block_size == 0) { /* LCOV_EXCL_START */ decoding_error(path, f); log_fatal("Zero 'blocksize' specification in the content file!\n"); @@ -2385,14 +2419,46 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* without configuration, auto assign the block size */ if (state->no_conf) { - state->block_size = blksize; + state->block_size = block_size; } - if (blksize != state->block_size) { + if (block_size != state->block_size) { /* LCOV_EXCL_START */ decoding_error(path, f); log_fatal("Mismatching 'blocksize' specification in the content file!\n"); - log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", blksize / KIBI); + log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } else if (c == 'y') { + uint32_t hash_size; + + ret = sgetb32(f, &hash_size); + if (ret < 0) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + os_abort(); + /* LCOV_EXCL_STOP */ + } + + if (hash_size < 2 || hash_size > HASH_MAX) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + log_fatal("Invalid 'hashsize' specification in the content file!\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* without configuration, auto assign the block size */ + if (state->no_conf) { + BLOCK_HASH_SIZE = hash_size; + } + + if ((int)hash_size != BLOCK_HASH_SIZE) { + /* LCOV_EXCL_START */ + decoding_error(path, f); + log_fatal("Mismatching 'hashsize' specification in the content file!\n"); + log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -2658,13 +2724,25 @@ static void* state_write_thread(void* arg) count_dir = 0; /* write header */ - swrite("SNAPCNT2\n\3\0\0", 12, f); + if (BLOCK_HASH_SIZE == HASH_MAX) { + swrite("SNAPCNT2\n\3\0\0", 12, f); + } else { + /* use version 3 only if the hash size is not standard */ + swrite("SNAPCNT3\n\3\0\0", 12, f); + } /* write block size and block max */ sputc('z', f); sputb32(state->block_size, f); sputc('x', f); sputb32(blockmax, f); + + /* write hash size only if not the standard one */ + if (BLOCK_HASH_SIZE != HASH_MAX) { + sputc('y', f); + sputb32(BLOCK_HASH_SIZE, f); + } + if (serror(f)) { /* LCOV_EXCL_START */ log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); @@ -4348,6 +4426,9 @@ void generate_configuration(const char* path) printf("# Use this blocksize\n"); printf("blocksize %u\n", state.block_size / KIBI); printf("\n"); + printf("# Use this hashsize\n"); + printf("hashsize %u\n", BLOCK_HASH_SIZE); + printf("\n"); for (i = 0; i < state.level; ++i) { printf("# Set the correct path for the %s file\n", lev_name(i)); if (state.parity[i].uuid[0]) diff --git a/snapraid.1 b/snapraid.1 index 6ccc31e7..84988ff1 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -1339,7 +1339,14 @@ block size in bytes. For example with 8 disks of 4 TB and a block size of 256 KiB (1 KiB = 1024 bytes) you have: .PP -RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB +RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.93 GiB +.PD 0 +.PP +.PD +.PP +Switching to a block size of 512 KiB, you have: +.PP +RAM = (8 * 4 * 10^12) * 17 / (512 * 2^10) = 0.97 GiB .PD 0 .PP .PD @@ -1384,6 +1391,33 @@ mkfs.ext4 \-m 0 \-T largefile4 DEVICE This results in about 1.5% of extra space. Meaning about 60 GB for a 4 TB disk, that allows about 460000 files in each data disk without any wasted space. +.SS hashsize SIZE_IN_BYTES +Defines the basic hash size in bytes of the saved blocks. +The default hashsize is 16 bytes \-> 128 bits, and it should work +for most cases. +.PP +A reason to use a different hashsize is if your system has less +than 1 GiB of memory. As a rule of thumb, with 1 GiB or more memory +use the default 16, with 512 MiB use 8, and with 256 GiB use 4. +.PP +In more details SnapRAID requires about TS*(1+HS)/BS bytes of RAM memory. +Where TS is the total size in bytes of your disk array, BS is the +block size in bytes, and HS is the hash size in bytes. +.PP +For example with 8 disks of 4 TB and a block size of 256 KiB +(1 KiB = 1024 bytes), and an hash size of 16, you have: +.PP +RAM = (8 * 4 * 10^12) * (1+16) / (256 * 2^10) = 1.93 GiB +.PD 0 +.PP +.PD +.PP +Switching to a hash size of 8, you have: +.PP +RAM = (8 * 4 * 10^12) * (1+8) / (256 * 2^10) = 1.02 GiB +.PD 0 +.PP +.PD .SS autosave SIZE_IN_GIGABYTES Automatically save the state when syncing or scrubbing after the specified amount of GB processed. diff --git a/snapraid.conf.example b/snapraid.conf.example index a22c478c..c6bbd6e5 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -51,10 +51,17 @@ exclude /tmp/ exclude /lost+found/ # Defines the block size in kibi bytes (1024 bytes) (uncomment to enable). +# WARNING: Changing this value is for experts only! # Default value is 256 -> 256 kibi bytes -> 262144 bytes # Format: "blocksize SIZE_IN_KiB" #blocksize 256 +# Defines the hash size in bytes (uncomment to enable). +# WARNING: Changing this value is for experts only! +# Default value is 16 -> 128 bits +# Format: "hashsize SIZE_IN_BYTES" +#hashsize 16 + # Automatically save the state when syncing after the specified amount # of GB processed (uncomment to enable). # This option is useful to avoid to restart from scratch long 'sync' diff --git a/snapraid.conf.example.windows b/snapraid.conf.example.windows index 7bd80d28..492ce2ba 100644 --- a/snapraid.conf.example.windows +++ b/snapraid.conf.example.windows @@ -55,10 +55,17 @@ exclude \Program Files (x86)\ exclude \Windows\ # Defines the block size in kibi bytes (1024 bytes) (uncomment to enable). +# WARNING: Changing this value is for experts only! # Default value is 256 -> 256 kibi bytes -> 262144 bytes # Format: "blocksize SIZE_IN_KiB" #blocksize 256 +# Defines the hash size in bytes (uncomment to enable). +# WARNING: Changing this value is for experts only! +# Default value is 16 -> 128 bits +# Format: "hashsize SIZE_IN_BYTES" +#hashsize 16 + # Automatically save the state when syncing after the specified amount # of GB processed (uncomment to enable). # This option is useful to avoid to restart from scratch long 'sync' diff --git a/snapraid.d b/snapraid.d index 24db64a3..cf5cf656 100644 --- a/snapraid.d +++ b/snapraid.d @@ -1043,7 +1043,11 @@ Configuration For example with 8 disks of 4 TB and a block size of 256 KiB (1 KiB = 1024 bytes) you have: - :RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB + :RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.93 GiB + + Switching to a block size of 512 KiB, you have: + + :RAM = (8 * 4 * 10^12) * 17 / (512 * 2^10) = 0.97 GiB Another reason to use a different blocksize is if you have a lot of small files. In the order of many millions. @@ -1081,6 +1085,28 @@ Configuration a 4 TB disk, that allows about 460000 files in each data disk without any wasted space. + hashsize SIZE_IN_BYTES + Defines the basic hash size in bytes of the saved blocks. + The default hashsize is 16 bytes -> 128 bits, and it should work + for most cases. + + A reason to use a different hashsize is if your system has less + than 1 GiB of memory. As a rule of thumb, with 1 GiB or more memory + use the default 16, with 512 MiB use 8, and with 256 GiB use 4. + + In more details SnapRAID requires about TS*(1+HS)/BS bytes of RAM memory. + Where TS is the total size in bytes of your disk array, BS is the + block size in bytes, and HS is the hash size in bytes. + + For example with 8 disks of 4 TB and a block size of 256 KiB + (1 KiB = 1024 bytes), and an hash size of 16, you have: + + :RAM = (8 * 4 * 10^12) * (1+16) / (256 * 2^10) = 1.93 GiB + + Switching to a hash size of 8, you have: + + :RAM = (8 * 4 * 10^12) * (1+8) / (256 * 2^10) = 1.02 GiB + autosave SIZE_IN_GIGABYTES Automatically save the state when syncing or scrubbing after the specified amount of GB processed. diff --git a/snapraid.txt b/snapraid.txt index d0ffc4ff..5c7316ca 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -1123,7 +1123,11 @@ block size in bytes. For example with 8 disks of 4 TB and a block size of 256 KiB (1 KiB = 1024 bytes) you have: -RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB +RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.93 GiB + +Switching to a block size of 512 KiB, you have: + +RAM = (8 * 4 * 10^12) * 17 / (512 * 2^10) = 0.97 GiB Another reason to use a different blocksize is if you have a lot of small files. In the order of many millions. @@ -1161,8 +1165,32 @@ This results in about 1.5% of extra space. Meaning about 60 GB for a 4 TB disk, that allows about 460000 files in each data disk without any wasted space. -7.9 autosave SIZE_IN_GIGABYTES ------------------------------- +7.9 hashsize SIZE_IN_BYTES +-------------------------- + +Defines the basic hash size in bytes of the saved blocks. +The default hashsize is 16 bytes -> 128 bits, and it should work +for most cases. + +A reason to use a different hashsize is if your system has less +than 1 GiB of memory. As a rule of thumb, with 1 GiB or more memory +use the default 16, with 512 MiB use 8, and with 256 GiB use 4. + +In more details SnapRAID requires about TS*(1+HS)/BS bytes of RAM memory. +Where TS is the total size in bytes of your disk array, BS is the +block size in bytes, and HS is the hash size in bytes. + +For example with 8 disks of 4 TB and a block size of 256 KiB +(1 KiB = 1024 bytes), and an hash size of 16, you have: + +RAM = (8 * 4 * 10^12) * (1+16) / (256 * 2^10) = 1.93 GiB + +Switching to a hash size of 8, you have: + +RAM = (8 * 4 * 10^12) * (1+8) / (256 * 2^10) = 1.02 GiB + +7.10 autosave SIZE_IN_GIGABYTES +------------------------------- Automatically save the state when syncing or scrubbing after the specified amount of GB processed. @@ -1170,7 +1198,7 @@ This option is useful to avoid to restart from scratch long "sync" commands interrupted by a machine crash, or any other event that may interrupt SnapRAID. -7.10 pool DIR +7.11 pool DIR ------------- Defines the pooling directory where the virtual view of the disk @@ -1178,7 +1206,7 @@ array is created using the "pool" command. The directory must already exist. -7.11 share UNC_DIR +7.12 share UNC_DIR ------------------ Defines the Windows UNC path required to access the disks remotely. @@ -1194,7 +1222,7 @@ file dir and name. This option is only required for Windows. -7.12 smartctl DISK/PARITY OPTIONS... +7.13 smartctl DISK/PARITY OPTIONS... ------------------------------------ Defines a custom smartctl command to obtain the SMART attributes @@ -1213,7 +1241,7 @@ Refers at the smartmontools documentation about the possible options: https://www.smartmontools.org/wiki/Supported_RAID-Controllers https://www.smartmontools.org/wiki/Supported_USB-Devices -7.13 Examples +7.14 Examples ------------- An example of a typical configuration for Unix is: