LCOV - code coverage report
Current view: top level - raid - internal.h (source / functions) Hit Total Coverage
Test: lcov.info Lines: 5 5 100.0 %
Date: 2015-05-12 Functions: 0 0 -

          Line data    Source code
       1             : /*
       2             :  * Copyright (C) 2013 Andrea Mazzoleni
       3             :  *
       4             :  * This program is free software: you can redistribute it and/or modify
       5             :  * it under the terms of the GNU General Public License as published by
       6             :  * the Free Software Foundation, either version 2 of the License, or
       7             :  * (at your option) any later version.
       8             :  *
       9             :  * This program is distributed in the hope that it will be useful,
      10             :  * but WITHOUT ANY WARRANTY; without even the implied warranty of
      11             :  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      12             :  * GNU General Public License for more details.
      13             :  */
      14             : 
      15             : #ifndef __RAID_INTERNAL_H
      16             : #define __RAID_INTERNAL_H
      17             : 
      18             : /*
      19             :  * Supported architecture.
      20             :  *
      21             :  * Detect the target architecture directly from the compiler.
      22             :  */
      23             : #if defined(__i386__)
      24             : #define CONFIG_X86 1
      25             : #define CONFIG_X86_32 1
      26             : #endif
      27             : 
      28             : #if defined(__x86_64__)
      29             : #define CONFIG_X86 1
      30             : #define CONFIG_X86_64 1
      31             : #endif
      32             : 
      33             : /*
      34             :  * Supported instruction sets.
      35             :  *
      36             :  * It may happen that the assembler is too old to support
      37             :  * all instructions, even if the architecture supports them.
      38             :  * These defines allow to exclude from the build the not supported ones.
      39             :  *
      40             :  * If in your project you use a predefined assembler, you can define them
      41             :  * using fixed values, instead of using the HAVE_* defines.
      42             :  */
      43             : #if HAVE_CONFIG_H
      44             : 
      45             : /* Includes the project configuration for HAVE_* defines */
      46             : #include "config.h"
      47             : 
      48             : #if HAVE_SSE2 /* Enables SSE2 only if the assembler supports it */
      49             : #define CONFIG_SSE2 1
      50             : #endif
      51             : 
      52             : #if HAVE_SSSE3 /* Enables SSSE3 only if the assembler supports it */
      53             : #define CONFIG_SSSE3 1
      54             : #endif
      55             : 
      56             : #if HAVE_AVX2 /* Enables AVX2 only if the assembler supports it */
      57             : #define CONFIG_AVX2 1
      58             : #endif
      59             : #else
      60             : /* If no config file, assumes the assembler supports everything in x86 */
      61             : #ifdef CONFIG_X86
      62             : #define CONFIG_SSE2 1
      63             : #define CONFIG_SSSE3 1
      64             : #define CONFIG_AVX2 1
      65             : #endif
      66             : #endif
      67             : 
      68             : /*
      69             :  * Includes anything required for compatibility.
      70             :  */
      71             : #include <assert.h>
      72             : #include <stdint.h>
      73             : #include <stdlib.h>
      74             : #include <string.h>
      75             : 
      76             : /*
      77             :  * Inverse assert.
      78             :  */
      79             : #define BUG_ON(a) assert(!(a))
      80             : 
      81             : /*
      82             :  * Forced inline.
      83             :  */
      84             : #ifndef __always_inline
      85             : #define __always_inline inline __attribute__((always_inline))
      86             : #endif
      87             : 
      88             : /*
      89             :  * Forced alignment.
      90             :  */
      91             : #ifndef __aligned
      92             : #define __aligned(a) __attribute__((aligned(a)))
      93             : #endif
      94             : 
      95             : /*
      96             :  * Align a pointer at the specified size.
      97             :  */
      98             : static __always_inline void *__align_ptr(void *ptr, uintptr_t size)
      99             : {
     100      680589 :         uintptr_t offset = (uintptr_t)ptr;
     101             : 
     102      680589 :         offset = (offset + size - 1U) & ~(size - 1U);
     103             : 
     104      680589 :         return (void *)offset;
     105             : }
     106             : 
     107             : /*
     108             :  * Includes the main interface headers.
     109             :  */
     110             : #include "raid.h"
     111             : #include "helper.h"
     112             : 
     113             : /*
     114             :  * Internal functions.
     115             :  *
     116             :  * These are intended to provide access for testing.
     117             :  */
     118             : int raid_selftest(void);
     119             : void raid_gen_ref(int nd, int np, size_t size, void **vv);
     120             : void raid_invert(uint8_t *M, uint8_t *V, int n);
     121             : void raid_delta_gen(int nr, int *id, int *ip, int nd, size_t size, void **v);
     122             : void raid_rec1of1(int *id, int nd, size_t size, void **v);
     123             : void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv);
     124             : void raid_gen1_int32(int nd, size_t size, void **vv);
     125             : void raid_gen1_int64(int nd, size_t size, void **vv);
     126             : void raid_gen1_sse2(int nd, size_t size, void **vv);
     127             : void raid_gen1_avx2(int nd, size_t size, void **vv);
     128             : void raid_gen2_int32(int nd, size_t size, void **vv);
     129             : void raid_gen2_int64(int nd, size_t size, void **vv);
     130             : void raid_gen2_sse2(int nd, size_t size, void **vv);
     131             : void raid_gen2_avx2(int nd, size_t size, void **vv);
     132             : void raid_gen2_sse2ext(int nd, size_t size, void **vv);
     133             : void raid_genz_int32(int nd, size_t size, void **vv);
     134             : void raid_genz_int64(int nd, size_t size, void **vv);
     135             : void raid_genz_sse2(int nd, size_t size, void **vv);
     136             : void raid_genz_sse2ext(int nd, size_t size, void **vv);
     137             : void raid_genz_avx2ext(int nd, size_t size, void **vv);
     138             : void raid_gen3_int8(int nd, size_t size, void **vv);
     139             : void raid_gen3_ssse3(int nd, size_t size, void **vv);
     140             : void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
     141             : void raid_gen3_avx2ext(int nd, size_t size, void **vv);
     142             : void raid_gen4_int8(int nd, size_t size, void **vv);
     143             : void raid_gen4_ssse3(int nd, size_t size, void **vv);
     144             : void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
     145             : void raid_gen4_avx2ext(int nd, size_t size, void **vv);
     146             : void raid_gen5_int8(int nd, size_t size, void **vv);
     147             : void raid_gen5_ssse3(int nd, size_t size, void **vv);
     148             : void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
     149             : void raid_gen5_avx2ext(int nd, size_t size, void **vv);
     150             : void raid_gen6_int8(int nd, size_t size, void **vv);
     151             : void raid_gen6_ssse3(int nd, size_t size, void **vv);
     152             : void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
     153             : void raid_gen6_avx2ext(int nd, size_t size, void **vv);
     154             : void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     155             : void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     156             : void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     157             : void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     158             : void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     159             : void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     160             : void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     161             : void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     162             : void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
     163             : 
     164             : /*
     165             :  * Internal naming.
     166             :  *
     167             :  * These are intented to provide access for testing.
     168             :  */
     169             : const char *raid_gen1_tag(void);
     170             : const char *raid_gen2_tag(void);
     171             : const char *raid_genz_tag(void);
     172             : const char *raid_gen3_tag(void);
     173             : const char *raid_gen4_tag(void);
     174             : const char *raid_gen5_tag(void);
     175             : const char *raid_gen6_tag(void);
     176             : const char *raid_rec1_tag(void);
     177             : const char *raid_rec2_tag(void);
     178             : const char *raid_recX_tag(void);
     179             : 
     180             : /*
     181             :  * Internal forwarders.
     182             :  */
     183             : extern void (*raid_gen3_ptr)(int nd, size_t size, void **vv);
     184             : extern void (*raid_genz_ptr)(int nd, size_t size, void **vv);
     185             : extern void (*raid_gen_ptr[RAID_PARITY_MAX])(
     186             :         int nd, size_t size, void **vv);
     187             : extern void (*raid_rec_ptr[RAID_PARITY_MAX])(
     188             :         int nr, int *id, int *ip, int nd, size_t size, void **vv);
     189             : 
     190             : /*
     191             :  * Tables.
     192             :  */
     193             : extern const uint8_t raid_gfmul[256][256] __aligned(256);
     194             : extern const uint8_t raid_gfexp[256] __aligned(256);
     195             : extern const uint8_t raid_gfinv[256] __aligned(256);
     196             : extern const uint8_t raid_gfvandermonde[3][256] __aligned(256);
     197             : extern const uint8_t raid_gfcauchy[6][256] __aligned(256);
     198             : extern const uint8_t raid_gfcauchypshufb[251][4][2][16] __aligned(256);
     199             : extern const uint8_t raid_gfmulpshufb[256][2][16] __aligned(256);
     200             : extern const uint8_t (*raid_gfgen)[256];
     201             : #define gfmul raid_gfmul
     202             : #define gfexp raid_gfexp
     203             : #define gfinv raid_gfinv
     204             : #define gfvandermonde raid_gfvandermonde
     205             : #define gfcauchy raid_gfcauchy
     206             : #define gfgenpshufb raid_gfcauchypshufb
     207             : #define gfmulpshufb raid_gfmulpshufb
     208             : #define gfgen raid_gfgen
     209             : 
     210             : /*
     211             :  * Assembler blocks.
     212             :  */
     213             : #ifdef CONFIG_X86
     214             : #ifdef CONFIG_SSE2
     215             : static __always_inline void raid_sse_begin(void)
     216             : {
     217             : }
     218             : 
     219             : static __always_inline void raid_sse_end(void)
     220             : {
     221             :         /* SSE and AVX code uses non-temporal writes, like MOVNTDQ, */
     222             :         /* that use a weak memory model. To ensure that other processors */
     223             :         /* see correctly the data written, we use a store-store memory */
     224             :         /* barrier at the end of the asm code */
     225     1349399 :         asm volatile ("sfence" : : : "memory");
     226             : 
     227             :         /* clobbers registers used in the asm code */
     228             :         /* this is required because in the Windows ABI, */
     229             :         /* registers xmm6-xmm15 should be kept by the callee. */
     230             :         /* this clobber list force the compiler to save any */
     231             :         /* register that needs to be saved */
     232             :         /* we check for __SSE2_ because we require that the */
     233             :         /* compiler supports SSE2 registers in the clobber list */
     234             : #ifdef __SSE2__
     235             :         asm volatile ("" : : : "%xmm0", "%xmm1", "%xmm2", "%xmm3");
     236             :         asm volatile ("" : : : "%xmm4", "%xmm5", "%xmm6", "%xmm7");
     237             : #ifdef CONFIG_X86_64
     238             :         asm volatile ("" : : : "%xmm8", "%xmm9", "%xmm10", "%xmm11");
     239             :         asm volatile ("" : : : "%xmm12", "%xmm13", "%xmm14", "%xmm15");
     240             : #endif
     241             : #endif
     242             : }
     243             : #endif
     244             : 
     245             : #ifdef CONFIG_AVX2
     246             : static __always_inline void raid_avx_begin(void)
     247             : {
     248             :         raid_sse_begin();
     249             : }
     250             : 
     251             : static __always_inline void raid_avx_end(void)
     252             : {
     253             :         raid_sse_end();
     254             : 
     255             :         /* reset the upper part of the ymm registers */
     256             :         /* to avoid the 70 clocks penality on the next */
     257             :         /* xmm register use */
     258       19675 :         asm volatile ("vzeroupper" : : : "memory");
     259             : }
     260             : #endif
     261             : #endif /* CONFIG_X86 */
     262             : 
     263             : #endif
     264             : 

Generated by: LCOV version 1.10