mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
Optimize hardware CRC32C using multiple lanes
This commit is contained in:
Andrea Mazzoleni
parent
787fc020e6
commit
15048ff14f
7 changed files with 398 additions and 136 deletions
|
|
@ -20,7 +20,6 @@ void test(int advise_mode)
|
|||
uint32_t u32 = -1L;
|
||||
uint64_t u64 = -1LL;
|
||||
uint32_t put_crc_stored;
|
||||
uint32_t put_crc_computed;
|
||||
|
||||
crc32c_init();
|
||||
|
||||
|
|
@ -79,13 +78,6 @@ void test(int advise_mode)
|
|||
}
|
||||
|
||||
put_crc_stored = scrc(s);
|
||||
put_crc_computed = scrc_stream(s);
|
||||
|
||||
if (put_crc_stored != put_crc_computed) {
|
||||
/* LCOV_EXCL_START */
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
if (sputble32(put_crc_stored, s) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
|
|
|
|||
|
|
@ -267,6 +267,7 @@ static struct crc_test_vector TEST_CRC32C[] = {
|
|||
{ "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff", 32, 0x62a8ab43 },
|
||||
{ "\x1f\x1e\x1d\x1c\x1b\x1a\x19\x18\x17\x16\x15\x14\x13\x12\x11\x10\x0f\x0e\x0d\x0c\x0b\x0a\x09\x08\x07\x06\x05\x04\x03\x02\x01\x00", 32, 0x113fdb5c },
|
||||
{ "\x01\xc0\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x14\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x14\x00\x00\x00\x18\x28\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00", 48, 0xd9963a56 },
|
||||
{ "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 16, 0xd9c908eb },
|
||||
{ "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f", 32, 0x46dd794e },
|
||||
{ "\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f\x20\x21\x22\x23\x24\x25\x26\x27\x28", 40, 0x0e2c157f },
|
||||
{ "\x29\x2a\x2b\x2c\x2d\x2e\x2f\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x3a\x3b\x3c\x3d\x3e\x3f\x40\x41\x42\x43\x44\x45\x46\x47\x48\x49\x4a\x4b\x4c\x4d\x4e\x4f\x50", 40, 0xe980ebf6 },
|
||||
|
|
@ -296,6 +297,74 @@ static void test_crc32c(void)
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
unsigned char data[65536 + 7];
|
||||
static const size_t size[] = {
|
||||
CRC32C_X86_64_BLOCK5_SIZE * 3 - 1,
|
||||
CRC32C_X86_64_BLOCK5_SIZE * 3,
|
||||
CRC32C_X86_64_BLOCK5_SIZE * 3 + 1,
|
||||
CRC32C_X86_64_BLOCK4_SIZE * 3 - 1,
|
||||
CRC32C_X86_64_BLOCK4_SIZE * 3,
|
||||
CRC32C_X86_64_BLOCK4_SIZE * 3 + 1,
|
||||
CRC32C_X86_64_BLOCK3_SIZE * 3 - 1,
|
||||
CRC32C_X86_64_BLOCK3_SIZE * 3,
|
||||
CRC32C_X86_64_BLOCK3_SIZE * 3 + 1,
|
||||
CRC32C_X86_64_BLOCK2_SIZE * 3 - 1,
|
||||
CRC32C_X86_64_BLOCK2_SIZE * 3,
|
||||
CRC32C_X86_64_BLOCK2_SIZE * 3 + 1,
|
||||
CRC32C_X86_64_BLOCK1_SIZE * 3 - 1,
|
||||
CRC32C_X86_64_BLOCK1_SIZE * 3,
|
||||
CRC32C_X86_64_BLOCK1_SIZE * 3 + 1,
|
||||
CRC32C_X86_64_BLOCK0_SIZE * 3 - 1,
|
||||
CRC32C_X86_64_BLOCK0_SIZE * 3,
|
||||
CRC32C_X86_64_BLOCK0_SIZE * 3 + 1,
|
||||
3 * (CRC32C_X86_64_BLOCK0_SIZE + CRC32C_X86_64_BLOCK1_SIZE + CRC32C_X86_64_BLOCK2_SIZE + CRC32C_X86_64_BLOCK3_SIZE + CRC32C_X86_64_BLOCK4_SIZE + CRC32C_X86_64_BLOCK5_SIZE) + 255,
|
||||
|
||||
CRC32C_ARM64_BLOCK4_SIZE * 2 - 1,
|
||||
CRC32C_ARM64_BLOCK4_SIZE * 2,
|
||||
CRC32C_ARM64_BLOCK4_SIZE * 2 + 1,
|
||||
CRC32C_ARM64_BLOCK5_SIZE * 2 - 1,
|
||||
CRC32C_ARM64_BLOCK5_SIZE * 2,
|
||||
CRC32C_ARM64_BLOCK5_SIZE * 2 + 1,
|
||||
CRC32C_ARM64_BLOCK3_SIZE * 2 - 1,
|
||||
CRC32C_ARM64_BLOCK3_SIZE * 2,
|
||||
CRC32C_ARM64_BLOCK3_SIZE * 2 + 1,
|
||||
CRC32C_ARM64_BLOCK2_SIZE * 2 - 1,
|
||||
CRC32C_ARM64_BLOCK2_SIZE * 2,
|
||||
CRC32C_ARM64_BLOCK2_SIZE * 2 + 1,
|
||||
CRC32C_ARM64_BLOCK1_SIZE * 2 - 1,
|
||||
CRC32C_ARM64_BLOCK1_SIZE * 2,
|
||||
CRC32C_ARM64_BLOCK1_SIZE * 2 + 1,
|
||||
CRC32C_ARM64_BLOCK0_SIZE * 2 - 1,
|
||||
CRC32C_ARM64_BLOCK0_SIZE * 2,
|
||||
CRC32C_ARM64_BLOCK0_SIZE * 2 + 1,
|
||||
2 * (CRC32C_ARM64_BLOCK0_SIZE + CRC32C_ARM64_BLOCK1_SIZE + CRC32C_ARM64_BLOCK2_SIZE + CRC32C_ARM64_BLOCK3_SIZE + CRC32C_ARM64_BLOCK4_SIZE + CRC32C_ARM64_BLOCK5_SIZE) + 255,
|
||||
65536
|
||||
};
|
||||
static const unsigned offset[] = { 0, 1, 3, 7 };
|
||||
unsigned j;
|
||||
unsigned k;
|
||||
|
||||
for (i = 0; i < sizeof(data); ++i)
|
||||
data[i] = (unsigned char)(i * 29 + i / 7);
|
||||
|
||||
for (j = 0; j < sizeof(size) / sizeof(size[0]); ++j) {
|
||||
for (k = 0; k < sizeof(offset) / sizeof(offset[0]); ++k) {
|
||||
uint32_t seed = 0x12345678;
|
||||
uint32_t digest = crc32c(seed, data + offset[k], size[j]);
|
||||
uint32_t digest_gen = crc32c_gen(seed, data + offset[k], size[j]);
|
||||
|
||||
if (digest != digest_gen) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed CRC32C large buffer test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
static void test_parity(void)
|
||||
|
|
|
|||
|
|
@ -3855,23 +3855,18 @@ struct state_write_thread_context {
|
|||
* Write all hashes in a contiguous run of file blocks.
|
||||
*
|
||||
* The source hashes are separated by the block state byte, so they cannot be
|
||||
* copied with a single memcpy(). Keep the stream pointer and the independent
|
||||
* CRC local while filling each buffered group to avoid the per-block stream
|
||||
* call and field updates. The CRC is still computed before each hash is copied,
|
||||
* preserving the memory-corruption check performed by swrite().
|
||||
* copied with a single memcpy(). Keep the stream pointer local while filling
|
||||
* each buffered group to avoid the per-block stream call and field updates.
|
||||
*/
|
||||
static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_off_t file_pos, block_off_t count)
|
||||
{
|
||||
unsigned char* block_ptr = (unsigned char*)file_block(file, file_pos);
|
||||
size_t block_stride = block_sizeof();
|
||||
|
||||
assert(f->flags & STREAM_FLAGS_CRC);
|
||||
|
||||
while (count) {
|
||||
size_t available = (size_t)(f->end - f->pos);
|
||||
block_off_t cached_count = available / BLOCK_HASH_SIZE;
|
||||
unsigned char* output;
|
||||
uint32_t crc;
|
||||
|
||||
if (cached_count > count)
|
||||
cached_count = count;
|
||||
|
|
@ -3888,18 +3883,15 @@ static int state_write_block_run(STREAM* f, struct snapraid_file* file, block_of
|
|||
}
|
||||
|
||||
output = f->pos;
|
||||
crc = f->crc_stream;
|
||||
for (block_off_t i = 0; i < cached_count; ++i) {
|
||||
struct snapraid_block* block = (struct snapraid_block*)block_ptr;
|
||||
|
||||
crc = crc32c_plain(crc, block->hash, BLOCK_HASH_SIZE);
|
||||
hash_copy(output, block->hash);
|
||||
|
||||
output += BLOCK_HASH_SIZE;
|
||||
block_ptr += block_stride;
|
||||
}
|
||||
|
||||
f->crc_stream = crc;
|
||||
f->pos = output;
|
||||
count -= cached_count;
|
||||
}
|
||||
|
|
@ -4473,19 +4465,6 @@ static void* state_write_thread(void* arg)
|
|||
/* get the file crc */
|
||||
crc = scrc(f);
|
||||
|
||||
/*
|
||||
* Compare the crc of the data written to file
|
||||
* with the one of the data written to the stream
|
||||
*/
|
||||
if (crc != scrc_stream(f)) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(ECONTENT, "CRC mismatch while writing the content stream.\n");
|
||||
log_fatal(ECONTENT, "DANGER! Your RAM memory is faulty! DO NOT PROCEED UNTIL FIXED!\n");
|
||||
log_fatal(ECONTENT, "Try running a memory test like http://www.memtest86.com/\n");
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
sputble32(crc, f);
|
||||
if (serror(f)) {
|
||||
/* LCOV_EXCL_START */
|
||||
|
|
|
|||
|
|
@ -68,7 +68,6 @@ STREAM* sopen_read(const char* file, int flags)
|
|||
s->offset_uncached = 0;
|
||||
s->crc = 0;
|
||||
s->crc_uncached = 0;
|
||||
s->crc_stream = CRC_IV;
|
||||
|
||||
return s;
|
||||
}
|
||||
|
|
@ -100,7 +99,6 @@ STREAM* sopen_multi_write(unsigned count, int flags)
|
|||
s->offset_uncached = 0;
|
||||
s->crc = 0;
|
||||
s->crc_uncached = 0;
|
||||
s->crc_stream = CRC_IV;
|
||||
|
||||
return s;
|
||||
}
|
||||
|
|
@ -389,12 +387,6 @@ uint32_t scrc(STREAM* s)
|
|||
return crc32c(s->crc_uncached, s->buffer, s->pos - s->buffer);
|
||||
}
|
||||
|
||||
uint32_t scrc_stream(STREAM* s)
|
||||
{
|
||||
assert(s->flags & STREAM_FLAGS_CRC);
|
||||
return s->crc_stream ^ CRC_IV;
|
||||
}
|
||||
|
||||
int sgetc_uncached(STREAM* s)
|
||||
{
|
||||
/* if at the end of the buffer, fill it */
|
||||
|
|
@ -735,10 +727,6 @@ int swrite_uncached(const void* void_data, size_t size, STREAM* f)
|
|||
if (available > size)
|
||||
available = size;
|
||||
|
||||
/* compute the independent CRC before copying to detect buffer corruption */
|
||||
if (f->flags & STREAM_FLAGS_CRC)
|
||||
f->crc_stream = crc32c_plain(f->crc_stream, data, available);
|
||||
|
||||
memcpy(f->pos, data, available);
|
||||
f->pos += available;
|
||||
data += available;
|
||||
|
|
|
|||
|
|
@ -65,26 +65,6 @@ struct stream {
|
|||
*/
|
||||
uint32_t crc_uncached;
|
||||
|
||||
/**
|
||||
* CRC of the data written to the stream.
|
||||
*
|
||||
* This is an extra check of the data that is written to
|
||||
* file to ensure that it's consistent even in case
|
||||
* of memory errors.
|
||||
*
|
||||
* This extra check takes about 2 seconds for each GB of
|
||||
* content file with the Intel CRC instruction,
|
||||
* and about 4 seconds without it.
|
||||
* But usually this doesn't slow down the write process,
|
||||
* as the disk is the bottle-neck.
|
||||
*
|
||||
* Note that this CRC doesn't have the IV processing.
|
||||
*
|
||||
* Not used in reading.
|
||||
* In writing, it's all the data wrote calling sput() functions.
|
||||
*/
|
||||
uint32_t crc_stream;
|
||||
|
||||
struct advise_struct* advise; /**< Cache advice state for each handle. */
|
||||
};
|
||||
|
||||
|
|
@ -147,11 +127,6 @@ int64_t stell(STREAM* s);
|
|||
*/
|
||||
uint32_t scrc(STREAM* s);
|
||||
|
||||
/**
|
||||
* Get the CRC of the processed data in put.
|
||||
*/
|
||||
uint32_t scrc_stream(STREAM* s);
|
||||
|
||||
/**
|
||||
* Check if the buffer has enough data loaded.
|
||||
*/
|
||||
|
|
@ -369,16 +344,6 @@ static inline int sputc(int c, STREAM* s)
|
|||
return -1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Update the crc *before* writing the data in the buffer
|
||||
*
|
||||
* This must be done before the memory write,
|
||||
* to be able to detect memory errors on the buffer,
|
||||
* happening before we write it on the file.
|
||||
*/
|
||||
if (s->flags & STREAM_FLAGS_CRC)
|
||||
s->crc_stream = crc32c_plain_char(s->crc_stream, c);
|
||||
|
||||
*s->pos++ = c;
|
||||
|
||||
return 0;
|
||||
|
|
@ -412,10 +377,6 @@ static __always_inline int swrite(const void* data, size_t size, STREAM* f)
|
|||
if (tommy_unlikely(!sptrlookup(f, size)))
|
||||
return swrite_uncached(data, size, f);
|
||||
|
||||
/* compute the independent CRC before copying to detect buffer corruption */
|
||||
if (f->flags & STREAM_FLAGS_CRC)
|
||||
f->crc_stream = crc32c_plain(f->crc_stream, data, size);
|
||||
|
||||
memcpy(f->pos, data, size);
|
||||
f->pos += size;
|
||||
|
||||
|
|
|
|||
104
cmdline/util.c
104
cmdline/util.c
|
|
@ -398,8 +398,88 @@ uint32_t CRC32C_3[256] = {
|
|||
0x4a21617b, 0x9764cbc3, 0xf54642fa, 0x2803e842
|
||||
};
|
||||
|
||||
#if CONFIG_X86
|
||||
int crc_x86;
|
||||
#ifdef CONFIG_X86_64
|
||||
uint32_t CRC32C_X86_64_SKIP[6][8][16];
|
||||
#endif
|
||||
|
||||
#if CONFIG_ARM_CRC
|
||||
uint32_t CRC32C_ARM64_SKIP[6][8][16];
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) || CONFIG_ARM_CRC
|
||||
static uint32_t crc32c_shift_table[sizeof(size_t) * 8][32];
|
||||
|
||||
static uint32_t crc32c_matrix_times(const uint32_t* matrix, uint32_t vector)
|
||||
{
|
||||
uint32_t value = 0;
|
||||
|
||||
while (vector) {
|
||||
if (vector & 1)
|
||||
value ^= *matrix;
|
||||
vector >>= 1;
|
||||
++matrix;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
static void crc32c_matrix_square(uint32_t* square, const uint32_t* matrix)
|
||||
{
|
||||
unsigned i;
|
||||
|
||||
for (i = 0; i < 32; ++i)
|
||||
square[i] = crc32c_matrix_times(matrix, matrix[i]);
|
||||
}
|
||||
|
||||
static void crc32c_shift_init(void)
|
||||
{
|
||||
uint32_t matrix[32];
|
||||
uint32_t square[32];
|
||||
uint32_t row;
|
||||
unsigned i;
|
||||
|
||||
/* reflected Castagnoli polynomial, represented as one zero-bit operator. */
|
||||
matrix[0] = 0x82f63b78;
|
||||
row = 1;
|
||||
for (i = 1; i < 32; ++i) {
|
||||
matrix[i] = row;
|
||||
row <<= 1;
|
||||
}
|
||||
|
||||
/* square three times to obtain the operator for one zero byte. */
|
||||
crc32c_matrix_square(square, matrix);
|
||||
crc32c_matrix_square(matrix, square);
|
||||
crc32c_matrix_square(crc32c_shift_table[0], matrix);
|
||||
|
||||
for (i = 1; i < sizeof(size_t) * 8; ++i)
|
||||
crc32c_matrix_square(crc32c_shift_table[i], crc32c_shift_table[i - 1]);
|
||||
}
|
||||
|
||||
uint32_t crc32c_shift(uint32_t crc, size_t size)
|
||||
{
|
||||
unsigned i = 0;
|
||||
|
||||
while (size) {
|
||||
if (size & 1)
|
||||
crc = crc32c_matrix_times(crc32c_shift_table[i], crc);
|
||||
size >>= 1;
|
||||
++i;
|
||||
}
|
||||
|
||||
return crc;
|
||||
}
|
||||
|
||||
static void crc32c_skip_init(uint32_t table[8][16], size_t size)
|
||||
{
|
||||
unsigned i;
|
||||
unsigned j;
|
||||
|
||||
/* Collapse each fixed-size shift into eight nibble table lookups. */
|
||||
for (i = 0; i < 8; ++i) {
|
||||
for (j = 0; j < 16; ++j)
|
||||
table[i][j] = crc32c_shift((uint32_t)j << (4 * i), size);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
uint32_t crc32c_gen(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
|
|
@ -446,10 +526,28 @@ void crc32c_init(void)
|
|||
crc32c = crc32c_gen;
|
||||
#if CONFIG_X86
|
||||
if (raid_cpu_has_crc32()) {
|
||||
crc_x86 = 1;
|
||||
#ifdef CONFIG_X86_64
|
||||
crc32c_shift_init();
|
||||
crc32c_skip_init(CRC32C_X86_64_SKIP[0], CRC32C_X86_64_BLOCK0_SIZE);
|
||||
crc32c_skip_init(CRC32C_X86_64_SKIP[1], CRC32C_X86_64_BLOCK1_SIZE);
|
||||
crc32c_skip_init(CRC32C_X86_64_SKIP[2], CRC32C_X86_64_BLOCK2_SIZE);
|
||||
crc32c_skip_init(CRC32C_X86_64_SKIP[3], CRC32C_X86_64_BLOCK3_SIZE);
|
||||
crc32c_skip_init(CRC32C_X86_64_SKIP[4], CRC32C_X86_64_BLOCK4_SIZE);
|
||||
crc32c_skip_init(CRC32C_X86_64_SKIP[5], CRC32C_X86_64_BLOCK5_SIZE);
|
||||
#endif
|
||||
crc32c = crc32c_x86;
|
||||
}
|
||||
#endif
|
||||
#if CONFIG_ARM_CRC
|
||||
crc32c_shift_init();
|
||||
crc32c_skip_init(CRC32C_ARM64_SKIP[0], CRC32C_ARM64_BLOCK0_SIZE);
|
||||
crc32c_skip_init(CRC32C_ARM64_SKIP[1], CRC32C_ARM64_BLOCK1_SIZE);
|
||||
crc32c_skip_init(CRC32C_ARM64_SKIP[2], CRC32C_ARM64_BLOCK2_SIZE);
|
||||
crc32c_skip_init(CRC32C_ARM64_SKIP[3], CRC32C_ARM64_BLOCK3_SIZE);
|
||||
crc32c_skip_init(CRC32C_ARM64_SKIP[4], CRC32C_ARM64_BLOCK4_SIZE);
|
||||
crc32c_skip_init(CRC32C_ARM64_SKIP[5], CRC32C_ARM64_BLOCK5_SIZE);
|
||||
crc32c = crc32c_arm64;
|
||||
#endif
|
||||
}
|
||||
|
||||
/****************************************************************************/
|
||||
|
|
|
|||
277
cmdline/util.h
277
cmdline/util.h
|
|
@ -91,31 +91,33 @@ extern uint32_t CRC32C_1[256];
|
|||
extern uint32_t CRC32C_2[256];
|
||||
extern uint32_t CRC32C_3[256];
|
||||
|
||||
/**
|
||||
* If the CPU support the CRC instructions.
|
||||
*/
|
||||
#if CONFIG_X86
|
||||
extern int crc_x86;
|
||||
#define CRC32C_X86_64_BLOCK0_SIZE 5456
|
||||
#define CRC32C_X86_64_BLOCK1_SIZE 2720
|
||||
#define CRC32C_X86_64_BLOCK2_SIZE 1360
|
||||
#define CRC32C_X86_64_BLOCK3_SIZE 672
|
||||
#define CRC32C_X86_64_BLOCK4_SIZE 336
|
||||
#define CRC32C_X86_64_BLOCK5_SIZE 160
|
||||
#ifdef CONFIG_X86_64
|
||||
extern uint32_t CRC32C_X86_64_SKIP[6][8][16];
|
||||
#endif
|
||||
|
||||
#define CRC32C_ARM64_BLOCK0_SIZE 8192
|
||||
#define CRC32C_ARM64_BLOCK1_SIZE 4096
|
||||
#define CRC32C_ARM64_BLOCK2_SIZE 2048
|
||||
#define CRC32C_ARM64_BLOCK3_SIZE 1024
|
||||
#define CRC32C_ARM64_BLOCK4_SIZE 512
|
||||
#define CRC32C_ARM64_BLOCK5_SIZE 256
|
||||
#if CONFIG_ARM_CRC
|
||||
extern uint32_t CRC32C_ARM64_SKIP[6][8][16];
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute CRC-32 (Castagnoli) for a single byte without the IV.
|
||||
* Advance a raw CRC-32C by a number of zero bytes.
|
||||
* Used to combine independent hardware CRC lanes.
|
||||
*/
|
||||
static inline uint32_t crc32c_plain_char(uint32_t crc, unsigned char c)
|
||||
{
|
||||
#if CONFIG_X86
|
||||
if (tommy_likely(crc_x86)) {
|
||||
asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (c));
|
||||
return crc;
|
||||
}
|
||||
#if defined(CONFIG_X86_64) || CONFIG_ARM_CRC
|
||||
uint32_t crc32c_shift(uint32_t crc, size_t size);
|
||||
#endif
|
||||
#if CONFIG_ARM_CRC
|
||||
asm ("crc32cb %w0, %w0, %w1\n" : "+r" (crc) : "r" (c));
|
||||
return crc;
|
||||
#else
|
||||
return CRC32C_0[(crc ^ c) & 0xff] ^ (crc >> 8);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CRC-32 (Castagnoli) without the IV.
|
||||
|
|
@ -142,27 +144,128 @@ static inline uint32_t crc32c_gen_plain(uint32_t crc, const unsigned char* ptr,
|
|||
* Compute the CRC-32 (Castagnoli) without the IV.
|
||||
*/
|
||||
#if CONFIG_X86
|
||||
static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
#ifdef CONFIG_X86_64
|
||||
static __always_inline uint32_t crc32c_x86_64_skip(uint32_t crc, const uint32_t table[8][16])
|
||||
{
|
||||
return table[0][crc & 0xf]
|
||||
^ table[1][(crc >> 4) & 0xf]
|
||||
^ table[2][(crc >> 8) & 0xf]
|
||||
^ table[3][(crc >> 12) & 0xf]
|
||||
^ table[4][(crc >> 16) & 0xf]
|
||||
^ table[5][(crc >> 20) & 0xf]
|
||||
^ table[6][(crc >> 24) & 0xf]
|
||||
^ table[7][crc >> 28];
|
||||
}
|
||||
|
||||
static __always_inline uint32_t crc32c_x86_64_3lane(uint32_t crc, const unsigned char* ptr, size_t lane_size, const uint32_t table[8][16])
|
||||
{
|
||||
const unsigned char* ptr0 = ptr;
|
||||
const unsigned char* ptr1 = ptr + lane_size;
|
||||
const unsigned char* ptr2 = ptr1 + lane_size;
|
||||
const unsigned char* end = ptr1;
|
||||
uint64_t crc0 = crc;
|
||||
uint64_t crc1 = 0;
|
||||
uint64_t crc2 = 0;
|
||||
|
||||
/*
|
||||
* Keep three dependency chains in flight to hide the CRC32 latency.
|
||||
* Lane sizes are multiples of 16 to avoid a separate tail loop.
|
||||
*/
|
||||
while (ptr0 < end) {
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc0) : "m" (*(const uint64_t*)ptr0));
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc1) : "m" (*(const uint64_t*)ptr1));
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc2) : "m" (*(const uint64_t*)ptr2));
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc0) : "m" (*(const uint64_t*)(ptr0 + 8)));
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc1) : "m" (*(const uint64_t*)(ptr1 + 8)));
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc2) : "m" (*(const uint64_t*)(ptr2 + 8)));
|
||||
ptr0 += 16;
|
||||
ptr1 += 16;
|
||||
ptr2 += 16;
|
||||
}
|
||||
|
||||
crc = crc32c_x86_64_skip((uint32_t)crc0, table) ^ (uint32_t)crc1;
|
||||
|
||||
return crc32c_x86_64_skip(crc, table) ^ (uint32_t)crc2;
|
||||
}
|
||||
#endif
|
||||
|
||||
static __always_inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
{
|
||||
#ifdef CONFIG_X86_64
|
||||
uint64_t crc64 = crc;
|
||||
while (size >= 8) {
|
||||
asm ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
|
||||
|
||||
if (size >= 3 * CRC32C_X86_64_BLOCK5_SIZE) {
|
||||
while (size >= 3 * CRC32C_X86_64_BLOCK0_SIZE) {
|
||||
crc64 = crc32c_x86_64_3lane((uint32_t)crc64, ptr, CRC32C_X86_64_BLOCK0_SIZE, CRC32C_X86_64_SKIP[0]);
|
||||
ptr += 3 * CRC32C_X86_64_BLOCK0_SIZE;
|
||||
size -= 3 * CRC32C_X86_64_BLOCK0_SIZE;
|
||||
}
|
||||
while (size >= 3 * CRC32C_X86_64_BLOCK1_SIZE) {
|
||||
crc64 = crc32c_x86_64_3lane((uint32_t)crc64, ptr, CRC32C_X86_64_BLOCK1_SIZE, CRC32C_X86_64_SKIP[1]);
|
||||
ptr += 3 * CRC32C_X86_64_BLOCK1_SIZE;
|
||||
size -= 3 * CRC32C_X86_64_BLOCK1_SIZE;
|
||||
}
|
||||
while (size >= 3 * CRC32C_X86_64_BLOCK2_SIZE) {
|
||||
crc64 = crc32c_x86_64_3lane((uint32_t)crc64, ptr, CRC32C_X86_64_BLOCK2_SIZE, CRC32C_X86_64_SKIP[2]);
|
||||
ptr += 3 * CRC32C_X86_64_BLOCK2_SIZE;
|
||||
size -= 3 * CRC32C_X86_64_BLOCK2_SIZE;
|
||||
}
|
||||
while (size >= 3 * CRC32C_X86_64_BLOCK3_SIZE) {
|
||||
crc64 = crc32c_x86_64_3lane((uint32_t)crc64, ptr, CRC32C_X86_64_BLOCK3_SIZE, CRC32C_X86_64_SKIP[3]);
|
||||
ptr += 3 * CRC32C_X86_64_BLOCK3_SIZE;
|
||||
size -= 3 * CRC32C_X86_64_BLOCK3_SIZE;
|
||||
}
|
||||
while (size >= 3 * CRC32C_X86_64_BLOCK4_SIZE) {
|
||||
crc64 = crc32c_x86_64_3lane((uint32_t)crc64, ptr, CRC32C_X86_64_BLOCK4_SIZE, CRC32C_X86_64_SKIP[4]);
|
||||
ptr += 3 * CRC32C_X86_64_BLOCK4_SIZE;
|
||||
size -= 3 * CRC32C_X86_64_BLOCK4_SIZE;
|
||||
}
|
||||
while (size >= 3 * CRC32C_X86_64_BLOCK5_SIZE) {
|
||||
crc64 = crc32c_x86_64_3lane((uint32_t)crc64, ptr, CRC32C_X86_64_BLOCK5_SIZE, CRC32C_X86_64_SKIP[5]);
|
||||
ptr += 3 * CRC32C_X86_64_BLOCK5_SIZE;
|
||||
size -= 3 * CRC32C_X86_64_BLOCK5_SIZE;
|
||||
}
|
||||
}
|
||||
while (size >= 16) {
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)(ptr + 8)));
|
||||
ptr += 16;
|
||||
size -= 16;
|
||||
}
|
||||
if (size >= 8) {
|
||||
asm volatile ("crc32q %1, %0\n" : "+r" (crc64) : "m" (*(const uint64_t*)ptr));
|
||||
ptr += 8;
|
||||
size -= 8;
|
||||
}
|
||||
crc = (uint32_t)crc64;
|
||||
#else
|
||||
while (size >= 4) {
|
||||
asm ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
|
||||
while (size >= 16) {
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4)));
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 8)));
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 12)));
|
||||
ptr += 16;
|
||||
size -= 16;
|
||||
}
|
||||
if (size >= 8) {
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)(ptr + 4)));
|
||||
ptr += 8;
|
||||
size -= 8;
|
||||
}
|
||||
#endif
|
||||
if (size >= 4) {
|
||||
asm volatile ("crc32l %1, %0\n" : "+r" (crc) : "m" (*(const uint32_t*)ptr));
|
||||
ptr += 4;
|
||||
size -= 4;
|
||||
}
|
||||
#endif
|
||||
while (size) {
|
||||
asm ("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr));
|
||||
++ptr;
|
||||
--size;
|
||||
if (size >= 2) {
|
||||
asm volatile ("crc32w %1, %0\n" : "+r" (crc) : "m" (*(const uint16_t*)ptr));
|
||||
ptr += 2;
|
||||
size -= 2;
|
||||
}
|
||||
if (size) {
|
||||
asm volatile ("crc32b %1, %0\n" : "+r" (crc) : "m" (*ptr));
|
||||
}
|
||||
|
||||
return crc;
|
||||
|
|
@ -170,24 +273,113 @@ static inline uint32_t crc32c_x86_plain(uint32_t crc, const unsigned char* ptr,
|
|||
#endif
|
||||
|
||||
#if CONFIG_ARM_CRC
|
||||
static inline uint32_t crc32c_arm64_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
static __always_inline uint32_t crc32c_arm64_skip(uint32_t crc, const uint32_t table[8][16])
|
||||
{
|
||||
return table[0][crc & 0xf]
|
||||
^ table[1][(crc >> 4) & 0xf]
|
||||
^ table[2][(crc >> 8) & 0xf]
|
||||
^ table[3][(crc >> 12) & 0xf]
|
||||
^ table[4][(crc >> 16) & 0xf]
|
||||
^ table[5][(crc >> 20) & 0xf]
|
||||
^ table[6][(crc >> 24) & 0xf]
|
||||
^ table[7][crc >> 28];
|
||||
}
|
||||
|
||||
static __always_inline uint32_t crc32c_arm64_2lane(uint32_t crc, const unsigned char* ptr, size_t lane_size, const uint32_t table[8][16])
|
||||
{
|
||||
const unsigned char* ptr0 = ptr;
|
||||
const unsigned char* ptr1 = ptr + lane_size;
|
||||
const unsigned char* end = ptr1;
|
||||
uint32_t crc0 = crc;
|
||||
uint32_t crc1 = 0;
|
||||
|
||||
/*
|
||||
* Keep two dependency chains in flight to hide the CRC32 latency.
|
||||
*/
|
||||
while (ptr0 < end) {
|
||||
uint64_t val00;
|
||||
uint64_t val01;
|
||||
uint64_t val02;
|
||||
uint64_t val03;
|
||||
uint64_t val10;
|
||||
uint64_t val11;
|
||||
uint64_t val12;
|
||||
uint64_t val13;
|
||||
|
||||
__builtin_memcpy(&val00, ptr0, 8);
|
||||
__builtin_memcpy(&val10, ptr1, 8);
|
||||
__builtin_memcpy(&val01, ptr0 + 8, 8);
|
||||
__builtin_memcpy(&val11, ptr1 + 8, 8);
|
||||
__builtin_memcpy(&val02, ptr0 + 16, 8);
|
||||
__builtin_memcpy(&val12, ptr1 + 16, 8);
|
||||
__builtin_memcpy(&val03, ptr0 + 24, 8);
|
||||
__builtin_memcpy(&val13, ptr1 + 24, 8);
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val00));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val10));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val01));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val11));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val02));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val12));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc0) : "r" (val03));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc1) : "r" (val13));
|
||||
ptr0 += 32;
|
||||
ptr1 += 32;
|
||||
}
|
||||
|
||||
return crc32c_arm64_skip(crc0, table) ^ crc1;
|
||||
}
|
||||
|
||||
static __always_inline uint32_t crc32c_arm64_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
{
|
||||
if (size >= 2 * CRC32C_ARM64_BLOCK5_SIZE) {
|
||||
while (size >= 2 * CRC32C_ARM64_BLOCK0_SIZE) {
|
||||
crc = crc32c_arm64_2lane(crc, ptr, CRC32C_ARM64_BLOCK0_SIZE, CRC32C_ARM64_SKIP[0]);
|
||||
ptr += 2 * CRC32C_ARM64_BLOCK0_SIZE;
|
||||
size -= 2 * CRC32C_ARM64_BLOCK0_SIZE;
|
||||
}
|
||||
while (size >= 2 * CRC32C_ARM64_BLOCK1_SIZE) {
|
||||
crc = crc32c_arm64_2lane(crc, ptr, CRC32C_ARM64_BLOCK1_SIZE, CRC32C_ARM64_SKIP[1]);
|
||||
ptr += 2 * CRC32C_ARM64_BLOCK1_SIZE;
|
||||
size -= 2 * CRC32C_ARM64_BLOCK1_SIZE;
|
||||
}
|
||||
while (size >= 2 * CRC32C_ARM64_BLOCK2_SIZE) {
|
||||
crc = crc32c_arm64_2lane(crc, ptr, CRC32C_ARM64_BLOCK2_SIZE, CRC32C_ARM64_SKIP[2]);
|
||||
ptr += 2 * CRC32C_ARM64_BLOCK2_SIZE;
|
||||
size -= 2 * CRC32C_ARM64_BLOCK2_SIZE;
|
||||
}
|
||||
while (size >= 2 * CRC32C_ARM64_BLOCK3_SIZE) {
|
||||
crc = crc32c_arm64_2lane(crc, ptr, CRC32C_ARM64_BLOCK3_SIZE, CRC32C_ARM64_SKIP[3]);
|
||||
ptr += 2 * CRC32C_ARM64_BLOCK3_SIZE;
|
||||
size -= 2 * CRC32C_ARM64_BLOCK3_SIZE;
|
||||
}
|
||||
while (size >= 2 * CRC32C_ARM64_BLOCK4_SIZE) {
|
||||
crc = crc32c_arm64_2lane(crc, ptr, CRC32C_ARM64_BLOCK4_SIZE, CRC32C_ARM64_SKIP[4]);
|
||||
ptr += 2 * CRC32C_ARM64_BLOCK4_SIZE;
|
||||
size -= 2 * CRC32C_ARM64_BLOCK4_SIZE;
|
||||
}
|
||||
while (size >= 2 * CRC32C_ARM64_BLOCK5_SIZE) {
|
||||
crc = crc32c_arm64_2lane(crc, ptr, CRC32C_ARM64_BLOCK5_SIZE, CRC32C_ARM64_SKIP[5]);
|
||||
ptr += 2 * CRC32C_ARM64_BLOCK5_SIZE;
|
||||
size -= 2 * CRC32C_ARM64_BLOCK5_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
while (size >= 8) {
|
||||
uint64_t val;
|
||||
__builtin_memcpy(&val, ptr, 8);
|
||||
asm ("crc32cx %w0, %w0, %x1\n" : "+r" (crc) : "r" (val));
|
||||
asm volatile ("crc32cx %w0, %w0, %x1\n" : "+r" (crc) : "r" (val));
|
||||
ptr += 8;
|
||||
size -= 8;
|
||||
}
|
||||
if (size >= 4) {
|
||||
uint32_t val;
|
||||
__builtin_memcpy(&val, ptr, 4);
|
||||
asm ("crc32cw %w0, %w0, %w1\n" : "+r" (crc) : "r" (val));
|
||||
asm volatile ("crc32cw %w0, %w0, %w1\n" : "+r" (crc) : "r" (val));
|
||||
ptr += 4;
|
||||
size -= 4;
|
||||
}
|
||||
while (size) {
|
||||
asm ("crc32cb %w0, %w0, %w1\n" : "+r" (crc) : "r" (*ptr));
|
||||
asm volatile ("crc32cb %w0, %w0, %w1\n" : "+r" (crc) : "r" (*ptr));
|
||||
++ptr;
|
||||
--size;
|
||||
}
|
||||
|
|
@ -196,23 +388,6 @@ static inline uint32_t crc32c_arm64_plain(uint32_t crc, const unsigned char* ptr
|
|||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Compute CRC-32 (Castagnoli) without the IV.
|
||||
*/
|
||||
static inline uint32_t crc32c_plain(uint32_t crc, const unsigned char* ptr, size_t size)
|
||||
{
|
||||
#if CONFIG_X86
|
||||
if (tommy_likely(crc_x86)) {
|
||||
return crc32c_x86_plain(crc, ptr, size);
|
||||
}
|
||||
#endif
|
||||
#if CONFIG_ARM_CRC
|
||||
return crc32c_arm64_plain(crc, ptr, size);
|
||||
#else
|
||||
return crc32c_gen_plain(crc, ptr, size);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the CRC-32 (Castagnoli)
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue