diff --git a/cmdline/elem.c b/cmdline/elem.c index 24662185..5d9f1108 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -758,6 +758,7 @@ void extent_free(void* void_extent) free(extent); } +__aligned(32) int extent_parity_compare(const void* void_a, const void* void_b) { const struct snapraid_extent* arg_a = void_a; @@ -771,6 +772,7 @@ int extent_parity_compare(const void* void_a, const void* void_b) return 0; } +__aligned(32) int extent_file_compare(const void* void_a, const void* void_b) { const struct snapraid_extent* arg_a = void_a; diff --git a/cmdline/elem.h b/cmdline/elem.h index faba91b2..81dd62a7 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -1410,6 +1410,7 @@ static inline snapraid_info info_get(tommy_arrayblkof* array, block_off_t pos) * Grow the array once and process each internal block directly, avoiding the * per-position grow and lookup performed by info_set(). */ +__aligned_loops(64) static inline void info_set_run(tommy_arrayblkof* array, block_off_t pos, block_off_t count, snapraid_info info) { tommy_arrayblkof_grow(array, pos + count); diff --git a/cmdline/state.c b/cmdline/state.c index 1a59a9e3..a21e96b4 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2294,6 +2294,7 @@ static void decoding_error(const char* path, STREAM* f) * pointer directly over cached hashes therefore preserves the same integrity * check performed by sread(). */ +__aligned_loops(64) static int state_read_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count, unsigned state, int discard_hash) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); @@ -3902,6 +3903,7 @@ struct state_write_context { * copied with a single memcpy(). Keep the stream pointer local while filling * each buffered group to avoid the per-block stream call and field updates. */ +__aligned_loops(64) static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count) { unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos); diff --git a/cmdline/util.c b/cmdline/util.c index e1f7ffe3..c6c2e1c4 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -453,6 +453,7 @@ uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, size_t size) } #if CONFIG_X86 +__aligned_loops(64) uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) { crc ^= CRC_IV; @@ -466,6 +467,7 @@ uint32_t crc32c_x86(uint32_t crc, const unsigned char* ptr, size_t size) #endif #if CONFIG_ARM_CRC +__aligned_loops(64) uint32_t crc32c_arm64(uint32_t crc, const unsigned char* ptr, size_t size) { crc ^= CRC_IV; diff --git a/cmdline/util.h b/cmdline/util.h index a197f01f..c3f282a7 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -148,6 +148,7 @@ static __always_inline uint32_t crc32c_x86_64_skip(uint32_t crc, const uint32_t ^ table[7][crc >> 28]; } +__aligned_loops(64) static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned char* ptr, size_t lane_size, const uint32_t table[8][16]) { const unsigned char* ptr0 = ptr; @@ -180,6 +181,7 @@ static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned } #endif +__aligned_loops(64) static __always_inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, size_t size) { #ifdef CONFIG_X86_64 diff --git a/os/portable.h b/os/portable.h index 4ed09558..6336600e 100644 --- a/os/portable.h +++ b/os/portable.h @@ -70,6 +70,22 @@ #define __noreturn __attribute__((noreturn)) #endif +#ifndef __aligned +#if defined(__GNUC__) || defined(__clang__) +#define __aligned(n) __attribute__((aligned(n))) +#else +#define __aligned(n) +#endif +#endif + +#ifndef __aligned_loops +#if defined(__GNUC__) && !defined(__clang__) +#define __aligned_loops(n) __attribute__((optimize("align-loops=" #n))) +#else +#define __aligned_loops(n) +#endif +#endif + /** * Architecture for inline assembly. */