diff --git a/HISTORY b/HISTORY index 2df3a42a..61b9521f 100644 --- a/HISTORY +++ b/HISTORY @@ -13,6 +13,15 @@ SnapRAID HISTORY file patterns containing real characters matching the globbing '*?[]' ones. In Unix it was already possible to do the same escaping with '\'. + * Added test options for selecting the file advise mode to use: + --test-io-advise-none for standard mode + --test-io-advise-sequential advise sequential operations (Linux/Windows) + --test-io-advise-flush flush data after every operation (Linux) + --test-io-advise-discard discard cache after every operation (Linux) + --test-io-advise-discard-2 discard cache every 2 MB (Linux) + --test-io-advise-discard-8 discard cache every 8 MB (Linux) + --test-io-advise-discard-32 discard cache every 32 MB (Linux) + --test-io-advise-direct use direct mode (Linux/Windows) * Fixed running in Windows platforms that miss the RtlGenRandom function. * Added a new 'hashsize' configuration option. It could be useful in diff --git a/Makefile.am b/Makefile.am index 58206254..c3d193b9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -436,7 +436,10 @@ endif $(FAILENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(PAR1) --test-expect-need-sync diff > output.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-fallocate -c $(PAR1) sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) status - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-sequential -c $(PAR1) check + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-io-advise-none -c $(PAR1) sync -F + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-io-advise-sequential -c $(PAR1) sync -F + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-io-advise-flush -c $(PAR1) sync -F + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-io-advise-discard -c $(PAR1) sync -F #### CHANGE LINKS #### # Use a different size ("22" instead of "1") to ensure to recognize the file different # even if it gets the same timestamp in case subsecond timestamp is no available diff --git a/cmdline/handle.c b/cmdline/handle.c index d936b101..e7fa682c 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -34,6 +34,7 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file, in return 0; } + advise_init(&handle->advise, mode); pathprint(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub); ret = mkancestor(handle->path); @@ -49,10 +50,7 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file, in /* flags for opening */ /* O_BINARY: open as binary file (Windows only) */ /* O_NOFOLLOW: do not follow links to ensure to open the real file */ - /* O_SEQUENTIAL: improve performance for sequential access (Windows only) */ - flags = O_BINARY | O_NOFOLLOW; - if ((mode & MODE_SEQUENTIAL) != 0) - flags |= O_SEQUENTIAL; + flags = O_BINARY | O_NOFOLLOW | advise_flags(&handle->advise); /* open for read write */ handle->f = open(handle->path, flags | O_RDWR); @@ -110,18 +108,13 @@ int handle_create(struct snapraid_handle* handle, struct snapraid_file* file, in /* get the size of the existing data */ handle->valid_size = handle->st.st_size; -#if HAVE_POSIX_FADVISE - if ((mode & MODE_SEQUENTIAL) != 0) { - /* advise sequential access */ - ret = posix_fadvise(handle->f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - log_fatal("Error advising file '%s'. %s.\n", handle->path, strerror(ret)); - return -1; - /* LCOV_EXCL_STOP */ - } + ret = advise_open(&handle->advise, handle->f); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error advising file '%s'. %s.\n", handle->path, strerror(errno)); + return -1; + /* LCOV_EXCL_STOP */ } -#endif return 0; } @@ -161,6 +154,7 @@ int handle_open(struct snapraid_handle* handle, struct snapraid_file* file, int return 0; } + advise_init(&handle->advise, mode); pathprint(handle->path, sizeof(handle->path), "%s%s", handle->disk->dir, file->sub); /* for sure not created */ @@ -169,10 +163,7 @@ int handle_open(struct snapraid_handle* handle, struct snapraid_file* file, int /* flags for opening */ /* O_BINARY: open as binary file (Windows only) */ /* O_NOFOLLOW: do not follow links to ensure to open the real file */ - /* O_SEQUENTIAL: improve performance for sequential access (Windows only) */ - flags = O_BINARY | O_NOFOLLOW; - if ((mode & MODE_SEQUENTIAL) != 0) - flags |= O_SEQUENTIAL; + flags = O_BINARY | O_NOFOLLOW | advise_flags(&handle->advise); /* open for read */ handle->f = open_noatime(handle->path, flags | O_RDONLY); @@ -204,18 +195,13 @@ int handle_open(struct snapraid_handle* handle, struct snapraid_file* file, int /* get the size of the existing data */ handle->valid_size = handle->st.st_size; -#if HAVE_POSIX_FADVISE - if ((mode & MODE_SEQUENTIAL) != 0) { - /* advise sequential access */ - ret = posix_fadvise(handle->f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - out("Error advising file '%s'. %s.\n", handle->path, strerror(ret)); - return -1; - /* LCOV_EXCL_STOP */ - } + ret = advise_open(&handle->advise, handle->f); + if (ret != 0) { + /* LCOV_EXCL_START */ + out("Error advising file '%s'. %s.\n", handle->path, strerror(errno)); + return -1; + /* LCOV_EXCL_STOP */ } -#endif return 0; } @@ -254,6 +240,7 @@ int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned c data_off_t offset; unsigned read_size; unsigned count; + int ret; offset = file_pos * (data_off_t)block_size; @@ -274,10 +261,11 @@ int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned c count = 0; do { - read_ret = pread(handle->f, block_buffer + count, read_size - count, offset + count); + /* read the full block to support O_DIRECT */ + read_ret = pread(handle->f, block_buffer + count, block_size - count, offset + count); if (read_ret < 0) { /* LCOV_EXCL_START */ - out("Error reading file '%s' at offset %" PRIu64 " for size %u. %s.\n", handle->path, offset + count, read_size - count, strerror(errno)); + out("Error reading file '%s' at offset %" PRIu64 " for size %u. %s.\n", handle->path, offset + count, block_size - count, strerror(errno)); return -1; /* LCOV_EXCL_STOP */ } @@ -294,16 +282,13 @@ int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned c memset(block_buffer + read_size, 0, block_size - read_size); } - /* Here isn't needed to call posix_fadvise(..., POSIX_FADV_DONTNEED) */ - /* because we already advised sequential access with POSIX_FADV_SEQUENTIAL. */ - /* In Linux 2.6.33 it's enough to ensure that data is not kept in the cache. */ - /* Better to do nothing and save a syscall for each block. */ - - /* Here we cannot call posix_fadvise(..., POSIX_FADV_WILLNEED) for the next block */ - /* because it may be blocking */ - /* See Ted Ts'o comment in "posix_fadvise(POSIX_FADV_WILLNEED) waits before returning?" */ - /* at: https://lkml.org/lkml/2010/12/6/122 */ - /* We relay only on the automatic readahead triggered by POSIX_FADV_SEQUENTIAL. */ + ret = advise_read(&handle->advise, handle->f, offset, block_size); + if (ret != 0) { + /* LCOV_EXCL_START */ + out("Error advising file '%s'. %s.\n", handle->path, strerror(errno)); + return -1; + /* LCOV_EXCL_STOP */ + } return read_size; } @@ -313,6 +298,7 @@ int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned ssize_t write_ret; data_off_t offset; unsigned write_size; + int ret; offset = file_pos * (data_off_t)block_size; @@ -331,11 +317,13 @@ int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned handle->valid_size = offset + write_size; } - /* Here doesn't make sense to call posix_fadvise(..., POSIX_FADV_DONTNEED) because */ - /* at this time the data is still not yet written and it cannot be discharged */ - /* from the system cache. */ - /* This is at least true for Linux 2.6.21 that just ignores POSIX_FADV_DONTNEED */ - /* if the data is not yet written. */ + ret = advise_write(&handle->advise, handle->f, offset, block_size); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error advising file '%s'. %s.\n", handle->path, strerror(errno)); + return -1; + /* LCOV_EXCL_STOP */ + } return 0; } diff --git a/cmdline/handle.h b/cmdline/handle.h index c8ca750b..f9521dbf 100644 --- a/cmdline/handle.h +++ b/cmdline/handle.h @@ -30,6 +30,7 @@ struct snapraid_handle { struct snapraid_file* file; /**< File opened. When the file is closed, it's set to 0. */ int f; /**< Handle of the file. */ struct stat st; /**< Stat info of the opened file. */ + struct advise_struct advise; /**< Advise information. */ data_off_t valid_size; /**< Size of the valid data. */ int created; /**< If the file was created, otherwise it was already existing. */ }; diff --git a/cmdline/import.c b/cmdline/import.c index f00a0910..91d42c51 100644 --- a/cmdline/import.c +++ b/cmdline/import.c @@ -62,6 +62,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t int f; int flags; unsigned block_size = state->block_size; + struct advise_struct advise; file = malloc_nofail(sizeof(struct snapraid_import_file)); file->path = strdup_nofail(path); @@ -71,11 +72,10 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t buffer = malloc_nofail(block_size); + advise_init(&advise, state->file_mode); + /* open for read */ - /* O_SEQUENTIAL: opening in sequential mode in Windows */ - flags = O_RDONLY | O_BINARY; - if ((state->file_mode & MODE_SEQUENTIAL) != 0) - flags |= O_SEQUENTIAL; + flags = O_RDONLY | O_BINARY | advise_flags(&advise); f = open(path, flags); if (f == -1) { /* LCOV_EXCL_START */ @@ -84,18 +84,13 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t /* LCOV_EXCL_STOP */ } -#if HAVE_POSIX_FADVISE - if ((state->file_mode & MODE_SEQUENTIAL) != 0) { - /* advise sequential access */ - ret = posix_fadvise(f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - log_fatal("Error advising file '%s'. %s.\n", path, strerror(ret)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } + ret = advise_open(&advise, f); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error advising file '%s'. %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ } -#endif offset = 0; for (i = 0; i < file->blockmax; ++i) { diff --git a/cmdline/io.c b/cmdline/io.c index 614a1a6a..8b43703c 100644 --- a/cmdline/io.c +++ b/cmdline/io.c @@ -880,7 +880,10 @@ void io_init(struct snapraid_io* io, struct snapraid_state* state, io->buffer_max = buffer_max; allocated = 0; for (i = 0; i < io->io_max; ++i) { - io->buffer_map[i] = malloc_nofail_vector_align(handle_max, buffer_max, state->block_size, &io->buffer_alloc_map[i]); + if (state->file_mode != ADVISE_DIRECT) + io->buffer_map[i] = malloc_nofail_vector_align(handle_max, buffer_max, state->block_size, &io->buffer_alloc_map[i]); + else + io->buffer_map[i] = malloc_nofail_vector_direct(handle_max, buffer_max, state->block_size, &io->buffer_alloc_map[i]); if (!state->opt.skip_self) mtest_vector(io->buffer_max, state->block_size, io->buffer_map[i]); allocated += state->block_size * buffer_max; diff --git a/cmdline/mingw.c b/cmdline/mingw.c index 554992af..5529d2df 100644 --- a/cmdline/mingw.c +++ b/cmdline/mingw.c @@ -602,6 +602,9 @@ static void windows_errno(DWORD error) /* Other cases are here identified with EINVAL */ errno = EINVAL; break; + case ERROR_HANDLE_EOF : /* in ReadFile() over the end of the file */ + errno = EINVAL; + break; case ERROR_FILE_NOT_FOUND : case ERROR_PATH_NOT_FOUND : /* in GetFileAttributeW() if internal path not found */ errno = ENOENT; @@ -1056,17 +1059,82 @@ FILE* windows_fopen(const char* file, const char* mode) int windows_open(const char* file, int flags, ...) { wchar_t conv_buf[CONV_MAX]; - va_list args; - int ret; + HANDLE h; + int f; + DWORD access; + DWORD share; + DWORD create; + DWORD attr; - va_start(args, flags); - if ((flags & O_CREAT) != 0) - ret = _wopen(convert(conv_buf, file), flags, va_arg(args, int)); - else - ret = _wopen(convert(conv_buf, file), flags); - va_end(args); + switch (flags & O_ACCMODE) { + case O_RDONLY : + access = GENERIC_READ; + break; + case O_WRONLY : + access = GENERIC_WRITE; + break; + case O_RDWR : + access = GENERIC_READ | GENERIC_WRITE; + break; + default: + errno = EINVAL; + return -1; + } - return ret; + share = FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE; + + switch (flags & (O_CREAT | O_EXCL | O_TRUNC)) { + case 0 : + create = OPEN_EXISTING; + break; + case O_CREAT : + create = OPEN_ALWAYS; + break; + case O_CREAT | O_EXCL : + case O_CREAT | O_EXCL | O_TRUNC : + create = CREATE_NEW; + break; + case O_CREAT | O_TRUNC : + create = CREATE_ALWAYS; + break; + case O_TRUNC : + create = TRUNCATE_EXISTING; + break; + default: + errno = EINVAL; + return -1; + } + + attr = FILE_ATTRIBUTE_NORMAL; + if ((flags & O_DIRECT) != 0) + attr |= FILE_FLAG_NO_BUFFERING; + if ((flags & O_DSYNC) != 0) + attr |= FILE_FLAG_WRITE_THROUGH; + if ((flags & O_RANDOM) != 0) + attr |= FILE_FLAG_RANDOM_ACCESS; + if ((flags & O_SEQUENTIAL) != 0) + attr |= FILE_FLAG_SEQUENTIAL_SCAN; + if ((flags & _O_SHORT_LIVED) != 0) + attr |= FILE_ATTRIBUTE_TEMPORARY; + if ((flags & O_TEMPORARY) != 0) + attr |= FILE_FLAG_DELETE_ON_CLOSE; + + h = CreateFileW(convert(conv_buf, file), access, share, 0, create, attr, 0); + if (h == INVALID_HANDLE_VALUE) { + windows_errno(GetLastError()); + return -1; + } + + /* mask out flags unknown by Windows */ + flags &= ~(O_DIRECT | O_DSYNC); + + f = _open_osfhandle((intptr_t)h, flags); + if (f == -1) { + CloseHandle(h); + return -1; + } + + return f; } struct windows_dir_struct { @@ -1785,6 +1853,22 @@ retry: return count; } +size_t windows_direct_size(void) +{ + SYSTEM_INFO si; + + GetSystemInfo(&si); + + /* + * MSDN 'File Buffering' + * https://msdn.microsoft.com/en-us/library/windows/desktop/cc644950%28v=vs.85%29.aspx + * + * "Therefore, in most situations, page-aligned memory will also be sector-aligned," + * "because the case where the sector size is larger than the page size is rare." + */ + return si.dwPageSize; +} + uint64_t tick(void) { LARGE_INTEGER t; diff --git a/cmdline/mingw.h b/cmdline/mingw.h index f36e4fe3..22b9acf4 100644 --- a/cmdline/mingw.h +++ b/cmdline/mingw.h @@ -119,6 +119,10 @@ #define pread windows_pread #undef pwrite #define pwrite windows_pwrite +#define direct_size windows_direct_size +#define HAVE_DIRECT_IO 1 +#define O_DIRECT 0x10000000 +#define O_DSYNC 0x20000000 /** * If nanoseconds are not supported, we report the special STAT_NSEC_INVALID value, @@ -362,6 +366,11 @@ ssize_t windows_pread(int f, void* buffer, size_t size, off_t offset); */ ssize_t windows_pwrite(int f, const void* buffer, size_t size, off_t offset); +/** + * List direct_size(). + */ +size_t windows_direct_size(void); + /****************************************************************************/ /* thread */ diff --git a/cmdline/parity.c b/cmdline/parity.c index cf726621..3b724192 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -190,17 +190,14 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p int ret; int flags; - /* mark it as initialized */ + advise_init(&split->advise, mode); pathcpy(split->path, sizeof(split->path), parity->split_map[s].path); split->f = -1; split->size = parity->split_map[s].size; ++handle->split_mac; /* opening in sequential mode in Windows */ - /* O_SEQUENTIAL: opening in sequential mode in Windows */ - flags = O_RDWR | O_CREAT | O_BINARY; - if ((mode & MODE_SEQUENTIAL) != 0) - flags |= O_SEQUENTIAL; + flags = O_RDWR | O_CREAT | O_BINARY | advise_flags(&split->advise); split->f = open(split->path, flags, 0600); if (split->f == -1) { /* LCOV_EXCL_START */ @@ -225,18 +222,13 @@ int parity_create(struct snapraid_parity_handle* handle, const struct snapraid_p if (split->size == PARITY_SIZE_INVALID) split->size = split->st.st_size; -#if HAVE_POSIX_FADVISE - if ((mode & MODE_SEQUENTIAL) != 0) { - /* advise sequential access */ - ret = posix_fadvise(split->f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(ret)); - goto bail; - /* LCOV_EXCL_STOP */ - } + ret = advise_open(&split->advise, split->f); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(errno)); + goto bail; + /* LCOV_EXCL_STOP */ } -#endif } return 0; @@ -520,18 +512,16 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par int ret; int flags; - /* mark it as initialized */ + advise_init(&split->advise, mode); pathcpy(split->path, sizeof(split->path), parity->split_map[s].path); split->f = -1; split->size = parity->split_map[s].size; ++handle->split_mac; /* open for read */ - /* O_SEQUENTIAL: opening in sequential mode in Windows */ /* O_NOATIME: do not change access time */ - flags = O_RDONLY | O_BINARY; - if ((mode & MODE_SEQUENTIAL) != 0) - flags |= O_SEQUENTIAL; + flags = O_RDONLY | O_BINARY | advise_flags(&split->advise); + split->f = open_noatime(split->path, flags); if (split->f == -1) { log_fatal("Error opening parity file '%s'. %s.\n", split->path, strerror(errno)); @@ -554,18 +544,13 @@ int parity_open(struct snapraid_parity_handle* handle, const struct snapraid_par if (split->size == PARITY_SIZE_INVALID) split->size = split->st.st_size; -#if HAVE_POSIX_FADVISE - if ((mode & MODE_SEQUENTIAL) != 0) { - /* advise sequential access */ - ret = posix_fadvise(split->f, 0, 0, POSIX_FADV_SEQUENTIAL); - if (ret != 0) { - /* LCOV_EXCL_START */ - log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(ret)); - goto bail; - /* LCOV_EXCL_STOP */ - } + ret = advise_open(&split->advise, split->f); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(errno)); + goto bail; + /* LCOV_EXCL_STOP */ } -#endif } return 0; @@ -659,6 +644,7 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne ssize_t write_ret; data_off_t offset; struct snapraid_split_handle* split; + int ret; offset = pos * (data_off_t)block_size; @@ -682,8 +668,13 @@ int parity_write(struct snapraid_parity_handle* handle, block_off_t pos, unsigne /* LCOV_EXCL_STOP */ } - /* Here doesn't make sense to call posix_fadvise(..., POSIX_FADV_DONTNEED) because */ - /* at this time the data is still in not yet written and it cannot be discharged. */ + ret = advise_write(&split->advise, split->f, offset, block_size); + if (ret != 0) { + /* LCOV_EXCL_START */ + log_fatal("Error advising parity file '%s'. %s.\n", split->path, strerror(errno)); + return -1; + /* LCOV_EXCL_STOP */ + } return 0; } @@ -694,6 +685,7 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned data_off_t offset; unsigned count; struct snapraid_split_handle* split; + int ret; offset = pos * (data_off_t)block_size; @@ -724,10 +716,13 @@ int parity_read(struct snapraid_parity_handle* handle, block_off_t pos, unsigned count += read_ret; } while (count < block_size); - /* Here isn't needed to call posix_fadvise(..., POSIX_FADV_DONTNEED) */ - /* because we already advised sequential access with POSIX_FADV_SEQUENTIAL. */ - /* In Linux 2.6.33 it's enough to ensure that data is not kept in the cache. */ - /* Better to do nothing and save a syscall for each block. */ + ret = advise_read(&split->advise, split->f, offset, block_size); + if (ret != 0) { + /* LCOV_EXCL_START */ + out("Error advising parity file '%s'. %s.\n", split->path, strerror(errno)); + return -1; + /* LCOV_EXCL_STOP */ + } return block_size; } diff --git a/cmdline/parity.h b/cmdline/parity.h index 44e0aa4d..b6eab8b7 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -27,6 +27,7 @@ struct snapraid_split_handle { char path[PATH_MAX]; /**< Path of the file. */ int f; /**< Handle of the files. */ struct stat st; /**< Stat info of the opened file. */ + struct advise_struct advise; /**< Advise information. */ /** * Size of the parity split. diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 38b89b1a..999aa9ee 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -254,7 +254,6 @@ void config(char* conf, size_t conf_size, const char* argv0) #define OPT_TEST_SKIP_SIGN 260 #define OPT_TEST_SKIP_FALLOCATE 261 #define OPT_TEST_SKIP_DEVICE 262 -#define OPT_TEST_SKIP_SEQUENTIAL 263 #define OPT_TEST_FORCE_MURMUR3 264 #define OPT_TEST_FORCE_SPOOKY2 265 #define OPT_TEST_SKIP_LOCK 266 @@ -283,6 +282,14 @@ void config(char* conf, size_t conf_size, const char* argv0) #define OPT_TEST_IO_CACHE 290 #define OPT_TEST_IO_STATS 291 #define OPT_TEST_COND_SIGNAL_OUTSIDE 292 +#define OPT_TEST_IO_ADVISE_NONE 293 +#define OPT_TEST_IO_ADVISE_SEQUENTIAL 294 +#define OPT_TEST_IO_ADVISE_FLUSH 295 +#define OPT_TEST_IO_ADVISE_DISCARD 296 +#define OPT_TEST_IO_ADVISE_DISCARD_2 297 +#define OPT_TEST_IO_ADVISE_DISCARD_8 298 +#define OPT_TEST_IO_ADVISE_DISCARD_32 299 +#define OPT_TEST_IO_ADVISE_DIRECT 300 #if HAVE_GETOPT_LONG struct option long_options[] = { @@ -335,9 +342,6 @@ struct option long_options[] = { /* Skip the fallocate() when growing the parity files */ { "test-skip-fallocate", 0, 0, OPT_TEST_SKIP_FALLOCATE }, - /* Skip the sequential hint when reading files */ - { "test-skip-sequential", 0, 0, OPT_TEST_SKIP_SEQUENTIAL }, - /* Skip the device check */ { "test-skip-device", 0, 0, OPT_TEST_SKIP_DEVICE }, @@ -419,6 +423,30 @@ struct option long_options[] = { /* Signal condition variable outside the mutex */ { "test-cond-signal-outside", 0, 0, OPT_TEST_COND_SIGNAL_OUTSIDE }, + /* Set the io advise to none */ + { "test-io-advise-none", 0, 0, OPT_TEST_IO_ADVISE_NONE }, + + /* Set the io advise to sequential */ + { "test-io-advise-sequential", 0, 0, OPT_TEST_IO_ADVISE_SEQUENTIAL }, + + /* Set the io advise to flush */ + { "test-io-advise-flush", 0, 0, OPT_TEST_IO_ADVISE_FLUSH }, + + /* Set the io advise to discard */ + { "test-io-advise-discard", 0, 0, OPT_TEST_IO_ADVISE_DISCARD }, + + /* Set the io advise to discard */ + { "test-io-advise-discard-2", 0, 0, OPT_TEST_IO_ADVISE_DISCARD_2 }, + + /* Set the io advise to discard */ + { "test-io-advise-discard-8", 0, 0, OPT_TEST_IO_ADVISE_DISCARD_8 }, + + /* Set the io advise to discard */ + { "test-io-advise-discard-32", 0, 0, OPT_TEST_IO_ADVISE_DISCARD_32 }, + + /* Set the io advise to direct */ + { "test-io-advise-direct", 0, 0, OPT_TEST_IO_ADVISE_DIRECT }, + { 0, 0, 0, 0 } }; #endif @@ -718,9 +746,6 @@ int main(int argc, char* argv[]) case OPT_TEST_SKIP_FALLOCATE : opt.skip_fallocate = 1; break; - case OPT_TEST_SKIP_SEQUENTIAL : - opt.skip_sequential = 1; - break; case OPT_TEST_SKIP_DEVICE : opt.skip_device = 1; period = 50; /* reduce period of the speed test */ @@ -818,6 +843,30 @@ int main(int argc, char* argv[]) thread_cond_signal_outside = 1; #endif break; + case OPT_TEST_IO_ADVISE_NONE : + opt.file_mode = ADVISE_NONE; + break; + case OPT_TEST_IO_ADVISE_SEQUENTIAL : + opt.file_mode = ADVISE_SEQUENTIAL; + break; + case OPT_TEST_IO_ADVISE_FLUSH : + opt.file_mode = ADVISE_FLUSH; + break; + case OPT_TEST_IO_ADVISE_DISCARD : + opt.file_mode = ADVISE_DISCARD; + break; + case OPT_TEST_IO_ADVISE_DISCARD_2 : + opt.file_mode = ADVISE_DISCARD_2; + break; + case OPT_TEST_IO_ADVISE_DISCARD_8 : + opt.file_mode = ADVISE_DISCARD_8; + break; + case OPT_TEST_IO_ADVISE_DISCARD_32 : + opt.file_mode = ADVISE_DISCARD_32; + break; + case OPT_TEST_IO_ADVISE_DIRECT : + opt.file_mode = ADVISE_DIRECT; + break; default : /* LCOV_EXCL_START */ log_fatal("Unknown option '%c'\n", (char)c); @@ -1074,6 +1123,20 @@ int main(int argc, char* argv[]) break; } + switch (operation) { +#if HAVE_DIRECT_IO + case OPERATION_SYNC : + case OPERATION_SCRUB : + case OPERATION_DRY : + break; +#endif + default: + /* we allow direct IO only on some commands */ + if (opt.file_mode == ADVISE_DIRECT) + opt.file_mode = ADVISE_SEQUENTIAL; + break; + } + switch (operation) { case OPERATION_DEVICES : case OPERATION_SMART : diff --git a/cmdline/state.c b/cmdline/state.c index 0c54b600..743905a8 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -147,7 +147,7 @@ void state_init(struct snapraid_state* state) state->checked_read = 0; state->block_size = 256 * KIBI; /* default 256 KiB */ state->raid_mode = RAID_MODE_CAUCHY; - state->file_mode = MODE_SEQUENTIAL; + state->file_mode = ADVISE_DEFAULT; for (l = 0; l < LEV_MAX; ++l) { state->parity[l].split_mac = 0; for (s = 0; s < SPLIT_MAX; ++s) { @@ -533,8 +533,12 @@ void state_config(struct snapraid_state* state, const char* path, const char* co state->opt.force_order = SORT_PHYSICAL; /* adjust file mode */ - if (state->opt.skip_sequential) - state->file_mode &= ~MODE_SEQUENTIAL; + if (state->opt.file_mode != ADVISE_DEFAULT) { + state->file_mode = state->opt.file_mode; + } else { + /* default mode, if nothing is specified */ + state->file_mode = ADVISE_SEQUENTIAL; + } /* store current command */ state->command = command; diff --git a/cmdline/state.h b/cmdline/state.h index 4f2d9322..b5f848d7 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -56,11 +56,6 @@ const char* lev_config_name(unsigned level); #define GIBI (1024 * 1024 * 1024) #define TEBI (1024 * 1024 * 1024 * 1024LL) -/** - * File modes. - */ -#define MODE_SEQUENTIAL 1 /**< Open the file in sequential mode. */ - /** * Global variable to identify if Ctrl+C is pressed. */ @@ -93,7 +88,7 @@ struct snapraid_option { int skip_device; /**< Skip devices matching checks. */ int skip_sign; /**< Skip the sign check for content files. */ int skip_fallocate; /**< Skip the use of fallocate(). */ - int skip_sequential; /**< Skip sequential hint. */ + int file_mode; /**< File mode. Mask of ADVISE_* flags. */ int skip_lock; /**< Skip the lock file protection. */ int skip_self; /**< Skip the self-test. */ int skip_content_check; /**< Relax some content file checks. */ @@ -129,7 +124,7 @@ struct snapraid_state { int checked_read; /**< If the state was read and checked. */ uint32_t block_size; /**< Block size in bytes. */ unsigned raid_mode; /**< Raid mode to use. RAID_MODE_DEFAULT or RAID_MODE_ALTERNATE. */ - int file_mode; /**< File access mode. Combination of MODE_* flags. */ + int file_mode; /**< File access mode. Combination of ADVISE_* flags. */ struct snapraid_parity parity[LEV_MAX]; /**< Parity vector. */ char share[PATH_MAX]; /**< Path of the share tree. If !=0 pool links are created in a different way. */ char pool[PATH_MAX]; /**< Path of the pool tree. */ diff --git a/cmdline/support.c b/cmdline/support.c index d41f9a2d..7a4b68db 100644 --- a/cmdline/support.c +++ b/cmdline/support.c @@ -879,6 +879,262 @@ int fmtime(int f, int64_t mtime_sec, int mtime_nsec) return ret; } +/****************************************************************************/ +/* advise */ + +void advise_init(struct advise_struct* advise, int mode) +{ + advise->mode = mode; + advise->dirty_begin = 0; + advise->dirty_end = 0; +} + +int advise_flags(struct advise_struct* advise) +{ + int flags = 0; + + if (advise->mode == ADVISE_SEQUENTIAL + || advise->mode == ADVISE_FLUSH + || advise->mode == ADVISE_DISCARD + || advise->mode == ADVISE_DISCARD_2 + || advise->mode == ADVISE_DISCARD_8 + || advise->mode == ADVISE_DISCARD_32 + ) + flags |= O_SEQUENTIAL; + +#if HAVE_DIRECT_IO + if (advise->mode == ADVISE_DIRECT) + flags |= O_DIRECT; +#endif + + return flags; +} + +int advise_open(struct advise_struct* advise, int f) +{ + (void)advise; + (void)f; + +#if HAVE_POSIX_FADVISE + if (advise->mode == ADVISE_SEQUENTIAL + || advise->mode == ADVISE_FLUSH + || advise->mode == ADVISE_DISCARD + || advise->mode == ADVISE_DISCARD_2 + || advise->mode == ADVISE_DISCARD_8 + || advise->mode == ADVISE_DISCARD_32 + ) { + int ret; + + /* advise sequential access */ + ret = posix_fadvise(f, 0, 0, POSIX_FADV_SEQUENTIAL); + if (ret != 0) { + /* LCOV_EXCL_START */ + errno = ret; /* posix_fadvise return the error code */ + return -1; + /* LCOV_EXCL_STOP */ + } + } +#endif + + return 0; +} + +int advise_write(struct advise_struct* advise, int f, size_t offset, size_t size) +{ + (void)advise; + (void)f; + (void)offset; + (void)size; + +#if HAVE_SYNC_FILE_RANGE + if (advise->mode == ADVISE_FLUSH) { + int ret; + + /* start writing immediately */ + ret = sync_file_range(f, offset, size, SYNC_FILE_RANGE_WRITE); + if (ret != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + } +#endif + + /* + * Follow Linus recommendations about fast writes. + * + * Linus "Unexpected splice "always copy" behavior observed" + * http://thread.gmane.org/gmane.linux.kernel/987247/focus=988070 + * --- + * I have had _very_ good experiences with even a rather trivial + * file writer that basically used (iirc) 8MB windows, and the logic was very + * trivial: + * + * - before writing a new 8M window, do "start writeback" + * (SYNC_FILE_RANGE_WRITE) on the previous window, and do + * a wait (SYNC_FILE_RANGE_WAIT_AFTER) on the window before that. + * + * in fact, in its simplest form, you can do it like this (this is from my + * "overwrite disk images" program that I use on old disks): + * + * for (index = 0; index < max_index ;index++) { + * if (write(fd, buffer, BUFSIZE) != BUFSIZE) + * break; + * // This won't block, but will start writeout asynchronously + * sync_file_range(fd, index*BUFSIZE, BUFSIZE, SYNC_FILE_RANGE_WRITE); + * // This does a blocking write-and-wait on any old ranges + * if (index) + * sync_file_range(fd, (index-1)*BUFSIZE, BUFSIZE, SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|SYNC_FILE_RANGE_WAIT_AFTER); + * } + * + * and even if you don't actually do a discard (maybe we should add a + * SYNC_FILE_RANGE_DISCARD bit, right now you'd need to do a separate + * fadvise(FADV_DONTNEED) to throw it out) the system behavior is pretty + * nice, because the heavy writer gets good IO performance _and_ leaves only + * easy-to-free pages around after itself. + * --- + * + * Linus "Unexpected splice "always copy" behavior observed" + * http://thread.gmane.org/gmane.linux.kernel/987247/focus=988176 + * --- + * The behavior for dirty page writeback is _not_ welldefined, and + * if you do POSIX_FADV_DONTNEED, I would suggest you do it as part of that + * writeback logic, ie you do it only on ranges that you have just waited on. + * + * IOW, in my example, you'd couple the + * + * sync_file_range(fd, (index-1)*BUFSIZE, BUFSIZE, SYNC_FILE_RANGE_WAIT_BEFORE|SYNC_FILE_RANGE_WRITE|SYNC_FILE_RANGE_WAIT_AFTER); + * + * with a + * + * posix_fadvise(fd, (index-1)*BUFSIZE, BUFSIZE, POSIX_FADV_DONTNEED); + * + * afterwards to throw out the pages that you just waited for. + * --- + */ + +#if HAVE_SYNC_FILE_RANGE && HAVE_POSIX_FADVISE + if (advise->mode == ADVISE_DISCARD + || advise->mode == ADVISE_DISCARD_2 + || advise->mode == ADVISE_DISCARD_8 + || advise->mode == ADVISE_DISCARD_32 + ) { + size_t dirty_offset = advise->dirty_begin; + size_t dirty_size = advise->dirty_end - advise->dirty_begin; + size_t limit; + + switch (advise->mode) { + default: + case ADVISE_DISCARD : limit = 1; break; + case ADVISE_DISCARD_2 : limit = 2 * 1024 * 1024; break; + case ADVISE_DISCARD_8 : limit = 8 * 1024 * 1024; break; + case ADVISE_DISCARD_32 : limit = 32 * 1024 * 1024; break; + } + + /* if we have to discard */ + if (dirty_size >= limit) { + int ret; + + /* send the data to the disk and wait until it's written */ + ret = sync_file_range(f, dirty_offset, dirty_size, SYNC_FILE_RANGE_WAIT_BEFORE | SYNC_FILE_RANGE_WRITE | SYNC_FILE_RANGE_WAIT_AFTER); + if (ret != 0) { + /* LCOV_EXCL_START */ + return -1; + /* LCOV_EXCL_STOP */ + } + + /* flush the data from the cache */ + ret = posix_fadvise(f, dirty_offset, dirty_size, POSIX_FADV_DONTNEED); + if (ret != 0) { + /* LCOV_EXCL_START */ + errno = ret; /* posix_fadvise return the error code */ + return -1; + /* LCOV_EXCL_STOP */ + } + + /* set the new dirty range */ + advise->dirty_begin = offset; + advise->dirty_end = offset + size; + } else { + /* extend the dirty range */ + if (advise->dirty_begin > offset) + advise->dirty_begin = offset; + if (advise->dirty_end < offset + size) + advise->dirty_end = offset + size; + } + } +#endif + + return 0; +} + +int advise_read(struct advise_struct* advise, int f, size_t offset, size_t size) +{ + (void)advise; + (void)f; + (void)offset; + (void)size; + +#if HAVE_POSIX_FADVISE + if (advise->mode == ADVISE_DISCARD + || advise->mode == ADVISE_DISCARD_2 + || advise->mode == ADVISE_DISCARD_8 + || advise->mode == ADVISE_DISCARD_32 + ) { + int ret; + + /* flush the data from the cache */ + ret = posix_fadvise(f, offset, size, POSIX_FADV_DONTNEED); + if (ret != 0) { + /* LCOV_EXCL_START */ + errno = ret; /* posix_fadvise return the error code */ + return -1; + /* LCOV_EXCL_STOP */ + } + } +#endif + + /* + * Here we cannot call posix_fadvise(..., POSIX_FADV_WILLNEED) for the next block + * because it may be blocking. + * + * Ted Ts'o "posix_fadvise(POSIX_FADV_WILLNEED) waits before returning?" + * https://lkml.org/lkml/2010/12/6/122 + * --- + * readahead and posix_fadvise(POSIX_FADV_WILLNEED) work exactly the same + * way, and in fact share mostly the same code path (see + * force_page_cache_readahead() in mm/readahead.c). + * + * They are asynchronous in that there is no guarantee the pages will be + * in the page cache by the time they return. But at the same time, they + * are not guaranteed to be non-blocking. That is, the work of doing the + * readahead does not take place in a kernel thread. So if you try to + * request I/O than will fit in the request queue, the system call will + * block until some I/O is completed so that more I/O requested cam be + * loaded onto the request queue. + * + * The only way to fix this would be to either put the work on a kernel + * thread (i.e., some kind of workqueue) or in a userspace thread. For + * ion programmer wondering what to do today, I'd suggest the + * latter since it will be more portable across various kernel versions. + * + * This does leave the question about whether we should change the kernel + * to allow readahead() and posix_fadvise(POSIX_FADV_WILLNEED) to be + * non-blocking and do this work in a workqueue (or via some kind of + * callback/continuation scheme). My worry is just doing this if a user + * application does something crazy, like request gigabytes and gigabytes + * of readahead, and then repents of their craziness, there should be a + * way of cancelling the readahead request. Today, the user can just + * kill the application. But if we simply shove the work to a kernel + * thread, it becomes a lot harder to cancel the readahead request. We'd + * have to invent a new API, and then have a way to know whether the user + * has access to kill a particular readahead request, etc. + * --- + */ + + return 0; +} + /****************************************************************************/ /* memory */ diff --git a/cmdline/support.h b/cmdline/support.h index 2c74320d..eafd7a20 100644 --- a/cmdline/support.h +++ b/cmdline/support.h @@ -322,6 +322,34 @@ int mkancestor(const char* file); */ int fmtime(int f, int64_t mtime_sec, int mtime_nsec); +/****************************************************************************/ +/* advise */ + +/** + * Advise modes. + */ +#define ADVISE_DEFAULT 0 /**< Default mode. */ +#define ADVISE_NONE 1 /**< Bare read/write mode. */ +#define ADVISE_SEQUENTIAL 2 /**< Sequential mode. */ +#define ADVISE_FLUSH 3 /**< Flush mode. */ +#define ADVISE_DISCARD 4 /**< Discard the cache after every operation. */ +#define ADVISE_DISCARD_2 5 /**< Discard the cache every 2MB. */ +#define ADVISE_DISCARD_8 6 /**< Discard the cache every 8MB. */ +#define ADVISE_DISCARD_32 7 /**< Discard the cache every 32MB. */ +#define ADVISE_DIRECT 8 /**< Direct mode. */ + +struct advise_struct { + int mode; + size_t dirty_begin; + size_t dirty_end; +}; + +void advise_init(struct advise_struct* advise, int mode); +int advise_flags(struct advise_struct* advise); +int advise_open(struct advise_struct* advise, int f); +int advise_write(struct advise_struct* advise, int f, size_t offset, size_t size); +int advise_read(struct advise_struct* advise, int f, size_t offset, size_t size); + /****************************************************************************/ /* memory */ diff --git a/cmdline/sync.c b/cmdline/sync.c index 91269001..960d29dd 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -35,6 +35,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta block_off_t i; unsigned j; void* buffer; + void* buffer_alloc; data_off_t countsize; block_off_t countpos; block_off_t countmax; @@ -49,7 +50,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta handle = handle_mapping(state, &diskmax); /* buffer for reading */ - buffer = malloc_nofail(state->block_size); + buffer = malloc_nofail_direct(state->block_size, &buffer_alloc); if (!state->opt.skip_self) mtest_vector(1, state->block_size, &buffer); @@ -383,7 +384,7 @@ bail: finish: free(handle); - free(buffer); + free(buffer_alloc); if (error + io_error + silent_error != 0) return -1; diff --git a/cmdline/unix.c b/cmdline/unix.c index d2cfc8e5..3fb2739d 100644 --- a/cmdline/unix.c +++ b/cmdline/unix.c @@ -1419,5 +1419,21 @@ void os_clear(void) printf("\033[2J"); /* clear screen */ } +size_t direct_size(void) +{ + long size; + + size = sysconf(_SC_PAGESIZE); + + if (size == -1) { + /* LCOV_EXCL_START */ + log_fatal("No page size\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + return size; +} + #endif diff --git a/cmdline/unix.h b/cmdline/unix.h index c4c6a749..575a9c4d 100644 --- a/cmdline/unix.h +++ b/cmdline/unix.h @@ -20,6 +20,7 @@ #ifdef __linux__ #define HAVE_LINUX_DEVICE 1 /**< In Linux enables special device support. */ +#define HAVE_DIRECT_IO 1 /**< Support O_DIRECT in open(). */ #endif #define O_BINARY 0 /**< Not used in Unix. */ @@ -57,5 +58,10 @@ int dirent_hidden(struct dirent* dd); */ const char* stat_desc(struct stat* st); +/** + * Return the aligment requirement for direct IO. + */ +size_t direct_size(void); + #endif diff --git a/cmdline/util.c b/cmdline/util.c index 444d3c8b..fda57f84 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -29,7 +29,23 @@ void* malloc_nofail_align(size_t size, void** freeptr) { void* ptr; - ptr = raid_malloc_align(size, freeptr); + ptr = raid_malloc(size, freeptr); + + if (!ptr) { + /* LCOV_EXCL_START */ + malloc_fail(size); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + return ptr; +} + +void* malloc_nofail_direct(size_t size, void** freeptr) +{ + void* ptr; + + ptr = raid_malloc_align(size, direct_size(), freeptr); if (!ptr) { /* LCOV_EXCL_START */ @@ -57,6 +73,22 @@ void** malloc_nofail_vector_align(int nd, int n, size_t size, void** freeptr) return ptr; } +void** malloc_nofail_vector_direct(int nd, int n, size_t size, void** freeptr) +{ + void* ptr; + + ptr = raid_malloc_vector_align(nd, n, size, direct_size(), 0, freeptr); + + if (!ptr) { + /* LCOV_EXCL_START */ + malloc_fail(n * size); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + return ptr; +} + void* malloc_nofail_test(size_t size) { void* ptr; diff --git a/cmdline/util.h b/cmdline/util.h index aa0ef9b3..f755e9d1 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -27,12 +27,23 @@ */ void* malloc_nofail_align(size_t size, void** freeptr); +/** + * Safe aligned malloc. Usable for direct io. + */ +void* malloc_nofail_direct(size_t size, void** freeptr); + /** * Safe aligned vector allocation. * If no memory is available, it aborts. */ void** malloc_nofail_vector_align(int nd, int n, size_t size, void** freeptr); +/** + * Safe page vector allocation. Usable for direct io. + * If no memory is available, it aborts. + */ +void** malloc_nofail_vector_direct(int nd, int n, size_t size, void** freeptr); + /** * Safe allocation with memory test. */ diff --git a/configure.ac b/configure.ac index 7abaa164..bbfaf87b 100644 --- a/configure.ac +++ b/configure.ac @@ -95,7 +95,8 @@ AC_CHECK_MEMBERS([struct statfs.f_fstypename], [], [], [[ dnl Checks for library functions. AC_CHECK_FUNCS([memset strchr strerror strrchr mkdir gettimeofday strtoul]) AC_CHECK_FUNCS([getopt getopt_long snprintf vsnprintf sigaction]) -AC_CHECK_FUNCS([ftruncate fallocate fsync access posix_fallocate posix_fadvise]) +AC_CHECK_FUNCS([ftruncate fallocate access]) +AC_CHECK_FUNCS([fsync posix_fadvise sync_file_range]) AC_CHECK_FUNCS([getc_unlocked ferror_unlocked fnmatch]) AC_CHECK_FUNCS([futimes futimens futimesat localtime_r]) AC_CHECK_FUNCS([fstatat flock statfs]) diff --git a/raid/memory.c b/raid/memory.c index ad82125d..02a5a927 100644 --- a/raid/memory.c +++ b/raid/memory.c @@ -15,12 +15,12 @@ #include "internal.h" #include "memory.h" -void *raid_malloc_align(size_t size, void **freeptr) +void *raid_malloc_align(size_t size, size_t align_size, void **freeptr) { unsigned char *ptr; uintptr_t offset; - ptr = malloc(size + RAID_MALLOC_ALIGN); + ptr = malloc(size + align_size); if (!ptr) { /* LCOV_EXCL_START */ return 0; @@ -29,15 +29,20 @@ void *raid_malloc_align(size_t size, void **freeptr) *freeptr = ptr; - offset = ((uintptr_t)ptr) % RAID_MALLOC_ALIGN; + offset = ((uintptr_t)ptr) % align_size; if (offset != 0) - ptr += RAID_MALLOC_ALIGN - offset; + ptr += align_size - offset; return ptr; } -void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) +void *raid_malloc(size_t size, void **freeptr) +{ + return raid_malloc_align(size, RAID_MALLOC_ALIGN, freeptr); +} + +void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, size_t displacement_size, void **freeptr) { void **v; unsigned char *va; @@ -52,7 +57,7 @@ void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) /* LCOV_EXCL_STOP */ } - va = raid_malloc_align(n * (size + RAID_MALLOC_DISPLACEMENT), freeptr); + va = raid_malloc_align(n * (size + displacement_size), align_size, freeptr); if (!va) { /* LCOV_EXCL_START */ free(v); @@ -62,7 +67,7 @@ void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) for (i = 0; i < n; ++i) { v[i] = va; - va += size + RAID_MALLOC_DISPLACEMENT; + va += size + displacement_size; } /* reverse order of the data blocks */ @@ -77,6 +82,11 @@ void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) return v; } +void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr) +{ + return raid_malloc_vector_align(nd, n, size, RAID_MALLOC_ALIGN, RAID_MALLOC_DISPLACEMENT, freeptr); +} + void raid_mrand_vector(unsigned seed, int n, size_t size, void **vv) { unsigned char **v = (unsigned char **)vv; diff --git a/raid/memory.h b/raid/memory.h index ff256c2b..de00614f 100644 --- a/raid/memory.h +++ b/raid/memory.h @@ -16,7 +16,7 @@ #define __RAID_MEMORY_H /** - * Memory alignment provided by raid_malloc_align(). + * Memory alignment provided by raid_malloc(). * * It should guarantee good cache performance everywhere. */ @@ -58,17 +58,29 @@ /** * Aligned malloc. + * Use an alignment suitable for the raid functions. */ -void *raid_malloc_align(size_t size, void **freeptr); +void *raid_malloc(size_t size, void **freeptr); + +/** + * Arbitrary aligned malloc. + */ +void *raid_malloc_align(size_t size, size_t align_size, void **freeptr); /** * Aligned vector allocation. + * Use an alignment suitable for the raid functions. * Returns a vector of @n pointers, each one pointing to a block of * the specified @size. * The first @nd elements are reversed in order. */ void **raid_malloc_vector(int nd, int n, size_t size, void **freeptr); +/** + * Arbitrary aligned vector allocation. + */ +void **raid_malloc_vector_align(int nd, int n, size_t size, size_t align_size, size_t displacement_size, void **freeptr); + /** * Fills the memory vector with pseudo-random data based on the specified seed. */