rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support a configurable hash size

Browse source
This commit is contained in:
Andrea Mazzoleni 2016-04-07 22:37:11 +02:00
parent 5b06549ee3
commit ab7d4e896c
6 changed files with 200 additions and 17 deletions
Download patch file Download diff file Expand all files Collapse all files

97
cmdline/state.c
View file

@ -609,6 +609,37 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* LCOV_EXCL_STOP */
}
state->block_size *= KIBI;
} else if (strcmp(tag, "hash_size") == 0) {
uint32_t hash_size;
ret = sgetu32(f, &hash_size);
if (ret < 0) {
/* LCOV_EXCL_START */
log_fatal("Invalid 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (hash_size < 2) {
/* LCOV_EXCL_START */
log_fatal("Too small 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (hash_size > HASH_MAX) {
/* LCOV_EXCL_START */
log_fatal("Too big 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* check if it's a power of 2 */
if ((hash_size & (hash_size - 1)) != 0) {
/* LCOV_EXCL_START */
log_fatal("Not power of 2 'hashsize' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
BLOCK_HASH_SIZE = hash_size;
} else if (lev_config_scan(tag, &level, &state->raid_mode) == 0) {
char device[PATH_MAX];
char* slash;
@ -1710,9 +1741,12 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
* - SNAPCNT2/SnapRAID 7.0 Adds entries 'M' and 'P', to add free_blocks support.
* The previous 'm' entry is now deprecated, but supported for importing.
* Similarly for text file, we add 'mapping' and 'parity' deprecating 'map'.
* - SNAPCNT3/SnapRAID 11.0 Adds entry 'y' for hash size.
*/
if (memcmp(buffer, "SNAPCNT1\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0) {
&& memcmp(buffer, "SNAPCNT2\n\3\0\0", 12) != 0
&& memcmp(buffer, "SNAPCNT3\n\3\0\0", 12) != 0
) {
/* LCOV_EXCL_START */
if (memcmp(buffer, "SNAPCNT", 7) != 0) {
decoding_error(path, f);
@ -2365,9 +2399,9 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* LCOV_EXCL_STOP */
}
} else if (c == 'z') {
block_off_t blksize;
uint32_t block_size;
ret = sgetb32(f, &blksize);
ret = sgetb32(f, &block_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
@ -2375,7 +2409,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* LCOV_EXCL_STOP */
}
if (blksize == 0) {
if (block_size == 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Zero 'blocksize' specification in the content file!\n");
@ -2385,14 +2419,46 @@ static void state_read_content(struct snapraid_state* state, const char* path, S
/* without configuration, auto assign the block size */
if (state->no_conf) {
state->block_size = blksize;
state->block_size = block_size;
}
if (blksize != state->block_size) {
if (block_size != state->block_size) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Mismatching 'blocksize' specification in the content file!\n");
log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", blksize / KIBI);
log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", block_size / KIBI);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
} else if (c == 'y') {
uint32_t hash_size;
ret = sgetb32(f, &hash_size);
if (ret < 0) {
/* LCOV_EXCL_START */
decoding_error(path, f);
os_abort();
/* LCOV_EXCL_STOP */
}
if (hash_size < 2 || hash_size > HASH_MAX) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Invalid 'hashsize' specification in the content file!\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
/* without configuration, auto assign the block size */
if (state->no_conf) {
BLOCK_HASH_SIZE = hash_size;
}
if ((int)hash_size != BLOCK_HASH_SIZE) {
/* LCOV_EXCL_START */
decoding_error(path, f);
log_fatal("Mismatching 'hashsize' specification in the content file!\n");
log_fatal("Please restore the 'hashsize' value in the configuration file to '%u'\n", hash_size);
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
@ -2658,13 +2724,25 @@ static void* state_write_thread(void* arg)
count_dir = 0;
/* write header */
swrite("SNAPCNT2\n\3\0\0", 12, f);
if (BLOCK_HASH_SIZE == HASH_MAX) {
swrite("SNAPCNT2\n\3\0\0", 12, f);
} else {
/* use version 3 only if the hash size is not standard */
swrite("SNAPCNT3\n\3\0\0", 12, f);
}
/* write block size and block max */
sputc('z', f);
sputb32(state->block_size, f);
sputc('x', f);
sputb32(blockmax, f);
/* write hash size only if not the standard one */
if (BLOCK_HASH_SIZE != HASH_MAX) {
sputc('y', f);
sputb32(BLOCK_HASH_SIZE, f);
}
if (serror(f)) {
/* LCOV_EXCL_START */
log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
@ -4348,6 +4426,9 @@ void generate_configuration(const char* path)
printf("# Use this blocksize\n");
printf("blocksize %u\n", state.block_size / KIBI);
printf("\n");
printf("# Use this hashsize\n");
printf("hashsize %u\n", BLOCK_HASH_SIZE);
printf("\n");
for (i = 0; i < state.level; ++i) {
printf("# Set the correct path for the %s file\n", lev_name(i));
if (state.parity[i].uuid[0])

36
snapraid.1
View file

@ -1339,7 +1339,14 @@ block size in bytes.
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes) you have:
.PP
RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB
RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.93 GiB
.PD 0
.PP
.PD
.PP
Switching to a block size of 512 KiB, you have:
.PP
RAM = (8 * 4 * 10^12) * 17 / (512 * 2^10) = 0.97 GiB
.PD 0
.PP
.PD
@ -1384,6 +1391,33 @@ mkfs.ext4 \-m 0 \-T largefile4 DEVICE
This results in about 1.5% of extra space. Meaning about 60 GB for
a 4 TB disk, that allows about 460000 files in each data disk without
any wasted space.
.SS hashsize SIZE_IN_BYTES
Defines the basic hash size in bytes of the saved blocks.
The default hashsize is 16 bytes \-> 128 bits, and it should work
for most cases.
.PP
A reason to use a different hashsize is if your system has less
than 1 GiB of memory. As a rule of thumb, with 1 GiB or more memory
use the default 16, with 512 MiB use 8, and with 256 GiB use 4.
.PP
In more details SnapRAID requires about TS*(1+HS)/BS bytes of RAM memory.
Where TS is the total size in bytes of your disk array, BS is the
block size in bytes, and HS is the hash size in bytes.
.PP
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes), and an hash size of 16, you have:
.PP
RAM = (8 * 4 * 10^12) * (1+16) / (256 * 2^10) = 1.93 GiB
.PD 0
.PP
.PD
.PP
Switching to a hash size of 8, you have:
.PP
RAM = (8 * 4 * 10^12) * (1+8) / (256 * 2^10) = 1.02 GiB
.PD 0
.PP
.PD
.SS autosave SIZE_IN_GIGABYTES
Automatically save the state when syncing or scrubbing after the specified amount
of GB processed.

7
snapraid.conf.example
View file

@ -51,10 +51,17 @@ exclude /tmp/
exclude /lost+found/
# Defines the block size in kibi bytes (1024 bytes) (uncomment to enable).
# WARNING: Changing this value is for experts only!
# Default value is 256 -> 256 kibi bytes -> 262144 bytes
# Format: "blocksize SIZE_IN_KiB"
#blocksize 256
# Defines the hash size in bytes (uncomment to enable).
# WARNING: Changing this value is for experts only!
# Default value is 16 -> 128 bits
# Format: "hashsize SIZE_IN_BYTES"
#hashsize 16
# Automatically save the state when syncing after the specified amount
# of GB processed (uncomment to enable).
# This option is useful to avoid to restart from scratch long 'sync'

7
snapraid.conf.example.windows
View file

@ -55,10 +55,17 @@ exclude \Program Files (x86)\
exclude \Windows\
# Defines the block size in kibi bytes (1024 bytes) (uncomment to enable).
# WARNING: Changing this value is for experts only!
# Default value is 256 -> 256 kibi bytes -> 262144 bytes
# Format: "blocksize SIZE_IN_KiB"
#blocksize 256
# Defines the hash size in bytes (uncomment to enable).
# WARNING: Changing this value is for experts only!
# Default value is 16 -> 128 bits
# Format: "hashsize SIZE_IN_BYTES"
#hashsize 16
# Automatically save the state when syncing after the specified amount
# of GB processed (uncomment to enable).
# This option is useful to avoid to restart from scratch long 'sync'

28
snapraid.d
View file

@ -1043,7 +1043,11 @@ Configuration
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes) you have:
:RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB
:RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.93 GiB
Switching to a block size of 512 KiB, you have:
:RAM = (8 * 4 * 10^12) * 17 / (512 * 2^10) = 0.97 GiB
Another reason to use a different blocksize is if you have a lot of
small files. In the order of many millions.
@ -1081,6 +1085,28 @@ Configuration
a 4 TB disk, that allows about 460000 files in each data disk without
any wasted space.
hashsize SIZE_IN_BYTES
Defines the basic hash size in bytes of the saved blocks.
The default hashsize is 16 bytes -> 128 bits, and it should work
for most cases.
A reason to use a different hashsize is if your system has less
than 1 GiB of memory. As a rule of thumb, with 1 GiB or more memory
use the default 16, with 512 MiB use 8, and with 256 GiB use 4.
In more details SnapRAID requires about TS*(1+HS)/BS bytes of RAM memory.
Where TS is the total size in bytes of your disk array, BS is the
block size in bytes, and HS is the hash size in bytes.
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes), and an hash size of 16, you have:
:RAM = (8 * 4 * 10^12) * (1+16) / (256 * 2^10) = 1.93 GiB
Switching to a hash size of 8, you have:
:RAM = (8 * 4 * 10^12) * (1+8) / (256 * 2^10) = 1.02 GiB
autosave SIZE_IN_GIGABYTES
Automatically save the state when syncing or scrubbing after the specified amount
of GB processed.

42
snapraid.txt
View file

@ -1123,7 +1123,11 @@ block size in bytes.
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes) you have:
RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB
RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.93 GiB
Switching to a block size of 512 KiB, you have:
RAM = (8 * 4 * 10^12) * 17 / (512 * 2^10) = 0.97 GiB
Another reason to use a different blocksize is if you have a lot of
small files. In the order of many millions.
@ -1161,8 +1165,32 @@ This results in about 1.5% of extra space. Meaning about 60 GB for
a 4 TB disk, that allows about 460000 files in each data disk without
any wasted space.
7.9 autosave SIZE_IN_GIGABYTES
------------------------------
7.9 hashsize SIZE_IN_BYTES
--------------------------
Defines the basic hash size in bytes of the saved blocks.
The default hashsize is 16 bytes -> 128 bits, and it should work
for most cases.
A reason to use a different hashsize is if your system has less
than 1 GiB of memory. As a rule of thumb, with 1 GiB or more memory
use the default 16, with 512 MiB use 8, and with 256 GiB use 4.
In more details SnapRAID requires about TS*(1+HS)/BS bytes of RAM memory.
Where TS is the total size in bytes of your disk array, BS is the
block size in bytes, and HS is the hash size in bytes.
For example with 8 disks of 4 TB and a block size of 256 KiB
(1 KiB = 1024 bytes), and an hash size of 16, you have:
RAM = (8 * 4 * 10^12) * (1+16) / (256 * 2^10) = 1.93 GiB
Switching to a hash size of 8, you have:
RAM = (8 * 4 * 10^12) * (1+8) / (256 * 2^10) = 1.02 GiB
7.10 autosave SIZE_IN_GIGABYTES
-------------------------------
Automatically save the state when syncing or scrubbing after the specified amount
of GB processed.
@ -1170,7 +1198,7 @@ This option is useful to avoid to restart from scratch long "sync"
commands interrupted by a machine crash, or any other event that
may interrupt SnapRAID.
7.10 pool DIR
7.11 pool DIR
-------------
Defines the pooling directory where the virtual view of the disk
@ -1178,7 +1206,7 @@ array is created using the "pool" command.
The directory must already exist.
7.11 share UNC_DIR
7.12 share UNC_DIR
------------------
Defines the Windows UNC path required to access the disks remotely.
@ -1194,7 +1222,7 @@ file dir and name.
This option is only required for Windows.
7.12 smartctl DISK/PARITY OPTIONS...
7.13 smartctl DISK/PARITY OPTIONS...
------------------------------------
Defines a custom smartctl command to obtain the SMART attributes
@ -1213,7 +1241,7 @@ Refers at the smartmontools documentation about the possible options:
https://www.smartmontools.org/wiki/Supported_RAID-Controllers
https://www.smartmontools.org/wiki/Supported_USB-Devices
7.13 Examples
7.14 Examples
-------------
An example of a typical configuration for Unix is: