mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
3399 lines
452 KiB
HTML
3399 lines
452 KiB
HTML
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
|
|
|
|
<html lang="en">
|
|
|
|
<head>
|
|
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
|
|
<title>LCOV - lcov.info - raid/x86.c</title>
|
|
<link rel="stylesheet" type="text/css" href="../gcov.css">
|
|
</head>
|
|
|
|
<body>
|
|
|
|
<table width="100%" border=0 cellspacing=0 cellpadding=0>
|
|
<tr><td class="title">LCOV - code coverage report</td></tr>
|
|
<tr><td class="ruler"><img src="../glass.png" width=3 height=3 alt=""></td></tr>
|
|
|
|
<tr>
|
|
<td width="100%">
|
|
<table cellpadding=1 border=0 width="100%">
|
|
<tr>
|
|
<td width="10%" class="headerItem">Current view:</td>
|
|
<td width="35%" class="headerValue"><a href="../index.html">top level</a> - <a href="index.html">raid</a> - x86.c<span style="font-size: 80%;"> (source / <a href="x86.c.func-sort-c.html">functions</a>)</span></td>
|
|
<td width="5%"></td>
|
|
<td width="15%"></td>
|
|
<td width="10%" class="headerCovTableHead">Hit</td>
|
|
<td width="10%" class="headerCovTableHead">Total</td>
|
|
<td width="15%" class="headerCovTableHead">Coverage</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="headerItem">Test:</td>
|
|
<td class="headerValue">lcov.info</td>
|
|
<td></td>
|
|
<td class="headerItem">Lines:</td>
|
|
<td class="headerCovTableEntry">2044</td>
|
|
<td class="headerCovTableEntry">2044</td>
|
|
<td class="headerCovTableEntryHi">100.0 %</td>
|
|
</tr>
|
|
<tr>
|
|
<td class="headerItem">Date:</td>
|
|
<td class="headerValue">2026-04-29 15:04:44</td>
|
|
<td></td>
|
|
<td class="headerItem">Functions:</td>
|
|
<td class="headerCovTableEntry">33</td>
|
|
<td class="headerCovTableEntry">33</td>
|
|
<td class="headerCovTableEntryHi">100.0 %</td>
|
|
</tr>
|
|
<tr><td><img src="../glass.png" width=3 height=3 alt=""></td></tr>
|
|
</table>
|
|
</td>
|
|
</tr>
|
|
|
|
<tr><td class="ruler"><img src="../glass.png" width=3 height=3 alt=""></td></tr>
|
|
</table>
|
|
|
|
<table cellpadding=0 cellspacing=0 border=0>
|
|
<tr>
|
|
<td><br></td>
|
|
</tr>
|
|
<tr>
|
|
<td>
|
|
<pre class="sourceHeading"> Line data Source code</pre>
|
|
<pre class="source">
|
|
<a name="1"><span class="lineNum"> 1 </span> : // SPDX-License-Identifier: GPL-2.0-or-later</a>
|
|
<a name="2"><span class="lineNum"> 2 </span> : // Copyright (C) 2013 Andrea Mazzoleni</a>
|
|
<a name="3"><span class="lineNum"> 3 </span> : </a>
|
|
<a name="4"><span class="lineNum"> 4 </span> : #include "internal.h"</a>
|
|
<a name="5"><span class="lineNum"> 5 </span> : #include "gf.h"</a>
|
|
<a name="6"><span class="lineNum"> 6 </span> : #include "cpu.h"</a>
|
|
<a name="7"><span class="lineNum"> 7 </span> : </a>
|
|
<a name="8"><span class="lineNum"> 8 </span> : /*</a>
|
|
<a name="9"><span class="lineNum"> 9 </span> : * For x86 optimizations you can see:</a>
|
|
<a name="10"><span class="lineNum"> 10 </span> : *</a>
|
|
<a name="11"><span class="lineNum"> 11 </span> : * Software optimization resources</a>
|
|
<a name="12"><span class="lineNum"> 12 </span> : * http://www.agner.org/optimize/</a>
|
|
<a name="13"><span class="lineNum"> 13 </span> : *</a>
|
|
<a name="14"><span class="lineNum"> 14 </span> : * x86, x64 Instruction Latency, Memory Latency and CPUID dumps</a>
|
|
<a name="15"><span class="lineNum"> 15 </span> : * http://users.atw.hu/instlatx64/</a>
|
|
<a name="16"><span class="lineNum"> 16 </span> : */</a>
|
|
<a name="17"><span class="lineNum"> 17 </span> : </a>
|
|
<a name="18"><span class="lineNum"> 18 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSE2)</a>
|
|
<a name="19"><span class="lineNum"> 19 </span> : /*</a>
|
|
<a name="20"><span class="lineNum"> 20 </span> : * GEN1 (RAID5 with xor) SSE2 implementation</a>
|
|
<a name="21"><span class="lineNum"> 21 </span> : *</a>
|
|
<a name="22"><span class="lineNum"> 22 </span> : * Intentionally don't process more than 64 bytes because 64 is the typical</a>
|
|
<a name="23"><span class="lineNum"> 23 </span> : * cache block, and processing 128 bytes doesn't increase performance, and in</a>
|
|
<a name="24"><span class="lineNum"> 24 </span> : * some cases it even decreases it.</a>
|
|
<a name="25"><span class="lineNum"> 25 </span> : */</a>
|
|
<a name="26"><span class="lineNum"> 26 </span><span class="lineCov"> 314473 : void raid_gen1_sse2(int nd, size_t size, void **vv)</span></a>
|
|
<a name="27"><span class="lineNum"> 27 </span> : {</a>
|
|
<a name="28"><span class="lineNum"> 28 </span><span class="lineCov"> 314473 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="29"><span class="lineNum"> 29 </span> : uint8_t *p;</a>
|
|
<a name="30"><span class="lineNum"> 30 </span> : int d, l;</a>
|
|
<a name="31"><span class="lineNum"> 31 </span> : size_t i;</a>
|
|
<a name="32"><span class="lineNum"> 32 </span> : </a>
|
|
<a name="33"><span class="lineNum"> 33 </span><span class="lineCov"> 314473 : l = nd - 1;</span></a>
|
|
<a name="34"><span class="lineNum"> 34 </span><span class="lineCov"> 314473 : p = v[nd];</span></a>
|
|
<a name="35"><span class="lineNum"> 35 </span> : </a>
|
|
<a name="36"><span class="lineNum"> 36 </span> : raid_sse_begin();</a>
|
|
<a name="37"><span class="lineNum"> 37 </span> : </a>
|
|
<a name="38"><span class="lineNum"> 38 </span><span class="lineCov"> 25851845 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="39"><span class="lineNum"> 39 </span><span class="lineCov"> 25537372 : asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="40"><span class="lineNum"> 40 </span><span class="lineCov"> 25537372 : asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));</span></a>
|
|
<a name="41"><span class="lineNum"> 41 </span><span class="lineCov"> 25537372 : asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));</span></a>
|
|
<a name="42"><span class="lineNum"> 42 </span><span class="lineCov"> 25537372 : asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48]));</span></a>
|
|
<a name="43"><span class="lineNum"> 43 </span><span class="lineCov"> 92504292 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="44"><span class="lineNum"> 44 </span><span class="lineCov"> 66966920 : asm volatile ("pxor %0,%%xmm0" : : "m" (v[d][i]));</span></a>
|
|
<a name="45"><span class="lineNum"> 45 </span><span class="lineCov"> 66966920 : asm volatile ("pxor %0,%%xmm1" : : "m" (v[d][i + 16]));</span></a>
|
|
<a name="46"><span class="lineNum"> 46 </span><span class="lineCov"> 66966920 : asm volatile ("pxor %0,%%xmm2" : : "m" (v[d][i + 32]));</span></a>
|
|
<a name="47"><span class="lineNum"> 47 </span><span class="lineCov"> 66966920 : asm volatile ("pxor %0,%%xmm3" : : "m" (v[d][i + 48]));</span></a>
|
|
<a name="48"><span class="lineNum"> 48 </span> : }</a>
|
|
<a name="49"><span class="lineNum"> 49 </span><span class="lineCov"> 25537372 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="50"><span class="lineNum"> 50 </span><span class="lineCov"> 25537372 : asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));</span></a>
|
|
<a name="51"><span class="lineNum"> 51 </span><span class="lineCov"> 25537372 : asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32]));</span></a>
|
|
<a name="52"><span class="lineNum"> 52 </span><span class="lineCov"> 25537372 : asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48]));</span></a>
|
|
<a name="53"><span class="lineNum"> 53 </span> : }</a>
|
|
<a name="54"><span class="lineNum"> 54 </span> : </a>
|
|
<a name="55"><span class="lineNum"> 55 </span> : raid_sse_end();</a>
|
|
<a name="56"><span class="lineNum"> 56 </span><span class="lineCov"> 314473 : }</span></a>
|
|
<a name="57"><span class="lineNum"> 57 </span> : #endif</a>
|
|
<a name="58"><span class="lineNum"> 58 </span> : </a>
|
|
<a name="59"><span class="lineNum"> 59 </span> : #if defined(CONFIG_X86) && defined(CONFIG_AVX2)</a>
|
|
<a name="60"><span class="lineNum"> 60 </span> : /*</a>
|
|
<a name="61"><span class="lineNum"> 61 </span> : * GEN1 (RAID5 with xor) AVX2 implementation</a>
|
|
<a name="62"><span class="lineNum"> 62 </span> : *</a>
|
|
<a name="63"><span class="lineNum"> 63 </span> : * Intentionally don't process more than 64 bytes because 64 is the typical</a>
|
|
<a name="64"><span class="lineNum"> 64 </span> : * cache block, and processing 128 bytes doesn't increase performance, and in</a>
|
|
<a name="65"><span class="lineNum"> 65 </span> : * some cases it even decreases it.</a>
|
|
<a name="66"><span class="lineNum"> 66 </span> : */</a>
|
|
<a name="67"><span class="lineNum"> 67 </span><span class="lineCov"> 26 : void raid_gen1_avx2(int nd, size_t size, void **vv)</span></a>
|
|
<a name="68"><span class="lineNum"> 68 </span> : {</a>
|
|
<a name="69"><span class="lineNum"> 69 </span><span class="lineCov"> 26 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="70"><span class="lineNum"> 70 </span> : uint8_t *p;</a>
|
|
<a name="71"><span class="lineNum"> 71 </span> : int d, l;</a>
|
|
<a name="72"><span class="lineNum"> 72 </span> : size_t i;</a>
|
|
<a name="73"><span class="lineNum"> 73 </span> : </a>
|
|
<a name="74"><span class="lineNum"> 74 </span><span class="lineCov"> 26 : l = nd - 1;</span></a>
|
|
<a name="75"><span class="lineNum"> 75 </span><span class="lineCov"> 26 : p = v[nd];</span></a>
|
|
<a name="76"><span class="lineNum"> 76 </span> : </a>
|
|
<a name="77"><span class="lineNum"> 77 </span> : raid_avx_begin();</a>
|
|
<a name="78"><span class="lineNum"> 78 </span> : </a>
|
|
<a name="79"><span class="lineNum"> 79 </span><span class="lineCov"> 94246 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="80"><span class="lineNum"> 80 </span><span class="lineCov"> 94220 : asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="81"><span class="lineNum"> 81 </span><span class="lineCov"> 94220 : asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32]));</span></a>
|
|
<a name="82"><span class="lineNum"> 82 </span><span class="lineCov"> 753924 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="83"><span class="lineNum"> 83 </span><span class="lineCov"> 659704 : asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (v[d][i]));</span></a>
|
|
<a name="84"><span class="lineNum"> 84 </span><span class="lineCov"> 659704 : asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (v[d][i + 32]));</span></a>
|
|
<a name="85"><span class="lineNum"> 85 </span> : }</a>
|
|
<a name="86"><span class="lineNum"> 86 </span><span class="lineCov"> 94220 : asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="87"><span class="lineNum"> 87 </span><span class="lineCov"> 94220 : asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));</span></a>
|
|
<a name="88"><span class="lineNum"> 88 </span> : }</a>
|
|
<a name="89"><span class="lineNum"> 89 </span> : </a>
|
|
<a name="90"><span class="lineNum"> 90 </span> : raid_avx_end();</a>
|
|
<a name="91"><span class="lineNum"> 91 </span><span class="lineCov"> 26 : }</span></a>
|
|
<a name="92"><span class="lineNum"> 92 </span> : #endif</a>
|
|
<a name="93"><span class="lineNum"> 93 </span> : </a>
|
|
<a name="94"><span class="lineNum"> 94 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="95"><span class="lineNum"> 95 </span> : /*</a>
|
|
<a name="96"><span class="lineNum"> 96 </span> : * GEN1 (RAID5 with xor) AVX512BW implementation</a>
|
|
<a name="97"><span class="lineNum"> 97 </span> : *</a>
|
|
<a name="98"><span class="lineNum"> 98 </span> : * Note that in true AVX512F would suffice, but we don't want to add</a>
|
|
<a name="99"><span class="lineNum"> 99 </span> : * specific support for AVX512F because this would be the only function</a>
|
|
<a name="100"><span class="lineNum"> 100 </span> : * to benefit from that.</a>
|
|
<a name="101"><span class="lineNum"> 101 </span> : *</a>
|
|
<a name="102"><span class="lineNum"> 102 </span> : * Intentionally don't process more than 64 bytes because 64 is the typical</a>
|
|
<a name="103"><span class="lineNum"> 103 </span> : * cache block, and processing 128 bytes doesn't increase performance, and in</a>
|
|
<a name="104"><span class="lineNum"> 104 </span> : * some cases it even decreases it.</a>
|
|
<a name="105"><span class="lineNum"> 105 </span> : */</a>
|
|
<a name="106"><span class="lineNum"> 106 </span><span class="lineCov"> 129 : void raid_gen1_avx512bw(int nd, size_t size, void **vv)</span></a>
|
|
<a name="107"><span class="lineNum"> 107 </span> : {</a>
|
|
<a name="108"><span class="lineNum"> 108 </span><span class="lineCov"> 129 : uint8_t **v = (uint8_t**)vv;</span></a>
|
|
<a name="109"><span class="lineNum"> 109 </span> : uint8_t *p;</a>
|
|
<a name="110"><span class="lineNum"> 110 </span> : int d, l;</a>
|
|
<a name="111"><span class="lineNum"> 111 </span> : size_t i;</a>
|
|
<a name="112"><span class="lineNum"> 112 </span> : </a>
|
|
<a name="113"><span class="lineNum"> 113 </span><span class="lineCov"> 129 : l = nd - 1;</span></a>
|
|
<a name="114"><span class="lineNum"> 114 </span><span class="lineCov"> 129 : p = v[nd];</span></a>
|
|
<a name="115"><span class="lineNum"> 115 </span> : </a>
|
|
<a name="116"><span class="lineNum"> 116 </span> : raid_avx_begin();</a>
|
|
<a name="117"><span class="lineNum"> 117 </span> : </a>
|
|
<a name="118"><span class="lineNum"> 118 </span><span class="lineCov"> 83085 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="119"><span class="lineNum"> 119 </span><span class="lineCov"> 82956 : asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="120"><span class="lineNum"> 120 </span><span class="lineCov"> 670468 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="121"><span class="lineNum"> 121 </span><span class="lineCov"> 587512 : asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (v[d][i]));</span></a>
|
|
<a name="122"><span class="lineNum"> 122 </span> : }</a>
|
|
<a name="123"><span class="lineNum"> 123 </span><span class="lineCov"> 82956 : asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="124"><span class="lineNum"> 124 </span> : }</a>
|
|
<a name="125"><span class="lineNum"> 125 </span> : </a>
|
|
<a name="126"><span class="lineNum"> 126 </span> : raid_avx_end();</a>
|
|
<a name="127"><span class="lineNum"> 127 </span><span class="lineCov"> 129 : }</span></a>
|
|
<a name="128"><span class="lineNum"> 128 </span> : #endif</a>
|
|
<a name="129"><span class="lineNum"> 129 </span> : </a>
|
|
<a name="130"><span class="lineNum"> 130 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSE2)</a>
|
|
<a name="131"><span class="lineNum"> 131 </span> : static const struct gfconst16 {</a>
|
|
<a name="132"><span class="lineNum"> 132 </span> : uint8_t poly[16];</a>
|
|
<a name="133"><span class="lineNum"> 133 </span> : uint8_t low4[16];</a>
|
|
<a name="134"><span class="lineNum"> 134 </span> : } gfconst16 __aligned(32) = {</a>
|
|
<a name="135"><span class="lineNum"> 135 </span> : {</a>
|
|
<a name="136"><span class="lineNum"> 136 </span> : 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d,</a>
|
|
<a name="137"><span class="lineNum"> 137 </span> : 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d</a>
|
|
<a name="138"><span class="lineNum"> 138 </span> : },</a>
|
|
<a name="139"><span class="lineNum"> 139 </span> : {</a>
|
|
<a name="140"><span class="lineNum"> 140 </span> : 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,</a>
|
|
<a name="141"><span class="lineNum"> 141 </span> : 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f</a>
|
|
<a name="142"><span class="lineNum"> 142 </span> : },</a>
|
|
<a name="143"><span class="lineNum"> 143 </span> : };</a>
|
|
<a name="144"><span class="lineNum"> 144 </span> : #endif</a>
|
|
<a name="145"><span class="lineNum"> 145 </span> : </a>
|
|
<a name="146"><span class="lineNum"> 146 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSE2)</a>
|
|
<a name="147"><span class="lineNum"> 147 </span> : /*</a>
|
|
<a name="148"><span class="lineNum"> 148 </span> : * GEN2 (RAID6 with powers of 2) SSE2 implementation</a>
|
|
<a name="149"><span class="lineNum"> 149 </span> : */</a>
|
|
<a name="150"><span class="lineNum"> 150 </span><span class="lineCov"> 29 : void raid_gen2_sse2(int nd, size_t size, void **vv)</span></a>
|
|
<a name="151"><span class="lineNum"> 151 </span> : {</a>
|
|
<a name="152"><span class="lineNum"> 152 </span><span class="lineCov"> 29 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="153"><span class="lineNum"> 153 </span> : uint8_t *p;</a>
|
|
<a name="154"><span class="lineNum"> 154 </span> : uint8_t *q;</a>
|
|
<a name="155"><span class="lineNum"> 155 </span> : int d, l;</a>
|
|
<a name="156"><span class="lineNum"> 156 </span> : size_t i;</a>
|
|
<a name="157"><span class="lineNum"> 157 </span> : </a>
|
|
<a name="158"><span class="lineNum"> 158 </span><span class="lineCov"> 29 : l = nd - 1;</span></a>
|
|
<a name="159"><span class="lineNum"> 159 </span><span class="lineCov"> 29 : p = v[nd];</span></a>
|
|
<a name="160"><span class="lineNum"> 160 </span><span class="lineCov"> 29 : q = v[nd + 1];</span></a>
|
|
<a name="161"><span class="lineNum"> 161 </span> : </a>
|
|
<a name="162"><span class="lineNum"> 162 </span> : raid_sse_begin();</a>
|
|
<a name="163"><span class="lineNum"> 163 </span> : </a>
|
|
<a name="164"><span class="lineNum"> 164 </span><span class="lineCov"> 29 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="165"><span class="lineNum"> 165 </span> : </a>
|
|
<a name="166"><span class="lineNum"> 166 </span><span class="lineCov"> 213045 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="167"><span class="lineNum"> 167 </span><span class="lineCov"> 213016 : asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="168"><span class="lineNum"> 168 </span><span class="lineCov"> 213016 : asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));</span></a>
|
|
<a name="169"><span class="lineNum"> 169 </span><span class="lineCov"> 213016 : asm volatile ("movdqa %xmm0,%xmm2");</span></a>
|
|
<a name="170"><span class="lineNum"> 170 </span><span class="lineCov"> 213016 : asm volatile ("movdqa %xmm1,%xmm3");</span></a>
|
|
<a name="171"><span class="lineNum"> 171 </span><span class="lineCov"> 1704456 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="172"><span class="lineNum"> 172 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm4,%xmm4");</span></a>
|
|
<a name="173"><span class="lineNum"> 173 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="174"><span class="lineNum"> 174 </span><span class="lineCov"> 1491440 : asm volatile ("pcmpgtb %xmm2,%xmm4");</span></a>
|
|
<a name="175"><span class="lineNum"> 175 </span><span class="lineCov"> 1491440 : asm volatile ("pcmpgtb %xmm3,%xmm5");</span></a>
|
|
<a name="176"><span class="lineNum"> 176 </span><span class="lineCov"> 1491440 : asm volatile ("paddb %xmm2,%xmm2");</span></a>
|
|
<a name="177"><span class="lineNum"> 177 </span><span class="lineCov"> 1491440 : asm volatile ("paddb %xmm3,%xmm3");</span></a>
|
|
<a name="178"><span class="lineNum"> 178 </span><span class="lineCov"> 1491440 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="179"><span class="lineNum"> 179 </span><span class="lineCov"> 1491440 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="180"><span class="lineNum"> 180 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="181"><span class="lineNum"> 181 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm5,%xmm3");</span></a>
|
|
<a name="182"><span class="lineNum"> 182 </span> : </a>
|
|
<a name="183"><span class="lineNum"> 183 </span><span class="lineCov"> 1491440 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="184"><span class="lineNum"> 184 </span><span class="lineCov"> 1491440 : asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));</span></a>
|
|
<a name="185"><span class="lineNum"> 185 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="186"><span class="lineNum"> 186 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="187"><span class="lineNum"> 187 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="188"><span class="lineNum"> 188 </span><span class="lineCov"> 1491440 : asm volatile ("pxor %xmm5,%xmm3");</span></a>
|
|
<a name="189"><span class="lineNum"> 189 </span> : }</a>
|
|
<a name="190"><span class="lineNum"> 190 </span><span class="lineCov"> 213016 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="191"><span class="lineNum"> 191 </span><span class="lineCov"> 213016 : asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));</span></a>
|
|
<a name="192"><span class="lineNum"> 192 </span><span class="lineCov"> 213016 : asm volatile ("movntdq %%xmm2,%0" : "=m" (q[i]));</span></a>
|
|
<a name="193"><span class="lineNum"> 193 </span><span class="lineCov"> 213016 : asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16]));</span></a>
|
|
<a name="194"><span class="lineNum"> 194 </span> : }</a>
|
|
<a name="195"><span class="lineNum"> 195 </span> : </a>
|
|
<a name="196"><span class="lineNum"> 196 </span> : raid_sse_end();</a>
|
|
<a name="197"><span class="lineNum"> 197 </span><span class="lineCov"> 29 : }</span></a>
|
|
<a name="198"><span class="lineNum"> 198 </span> : #endif</a>
|
|
<a name="199"><span class="lineNum"> 199 </span> : </a>
|
|
<a name="200"><span class="lineNum"> 200 </span> : #if defined(CONFIG_X86) && defined(CONFIG_AVX2)</a>
|
|
<a name="201"><span class="lineNum"> 201 </span> : /*</a>
|
|
<a name="202"><span class="lineNum"> 202 </span> : * GEN2 (RAID6 with powers of 2) AVX2 implementation</a>
|
|
<a name="203"><span class="lineNum"> 203 </span> : */</a>
|
|
<a name="204"><span class="lineNum"> 204 </span><span class="lineCov"> 19 : void raid_gen2_avx2(int nd, size_t size, void **vv)</span></a>
|
|
<a name="205"><span class="lineNum"> 205 </span> : {</a>
|
|
<a name="206"><span class="lineNum"> 206 </span><span class="lineCov"> 19 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="207"><span class="lineNum"> 207 </span> : uint8_t *p;</a>
|
|
<a name="208"><span class="lineNum"> 208 </span> : uint8_t *q;</a>
|
|
<a name="209"><span class="lineNum"> 209 </span> : int d, l;</a>
|
|
<a name="210"><span class="lineNum"> 210 </span> : size_t i;</a>
|
|
<a name="211"><span class="lineNum"> 211 </span> : </a>
|
|
<a name="212"><span class="lineNum"> 212 </span><span class="lineCov"> 19 : l = nd - 1;</span></a>
|
|
<a name="213"><span class="lineNum"> 213 </span><span class="lineCov"> 19 : p = v[nd];</span></a>
|
|
<a name="214"><span class="lineNum"> 214 </span><span class="lineCov"> 19 : q = v[nd + 1];</span></a>
|
|
<a name="215"><span class="lineNum"> 215 </span> : </a>
|
|
<a name="216"><span class="lineNum"> 216 </span> : raid_avx_begin();</a>
|
|
<a name="217"><span class="lineNum"> 217 </span> : </a>
|
|
<a name="218"><span class="lineNum"> 218 </span><span class="lineCov"> 19 : asm volatile ("vbroadcasti128 %0, %%ymm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="219"><span class="lineNum"> 219 </span><span class="lineCov"> 19 : asm volatile ("vpxor %ymm6,%ymm6,%ymm6");</span></a>
|
|
<a name="220"><span class="lineNum"> 220 </span> : </a>
|
|
<a name="221"><span class="lineNum"> 221 </span><span class="lineCov"> 65567 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="222"><span class="lineNum"> 222 </span><span class="lineCov"> 65548 : asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="223"><span class="lineNum"> 223 </span><span class="lineCov"> 65548 : asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32]));</span></a>
|
|
<a name="224"><span class="lineNum"> 224 </span><span class="lineCov"> 65548 : asm volatile ("vmovdqa %ymm0,%ymm2");</span></a>
|
|
<a name="225"><span class="lineNum"> 225 </span><span class="lineCov"> 65548 : asm volatile ("vmovdqa %ymm1,%ymm3");</span></a>
|
|
<a name="226"><span class="lineNum"> 226 </span><span class="lineCov"> 524548 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="227"><span class="lineNum"> 227 </span><span class="lineCov"> 459000 : asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4");</span></a>
|
|
<a name="228"><span class="lineNum"> 228 </span><span class="lineCov"> 459000 : asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5");</span></a>
|
|
<a name="229"><span class="lineNum"> 229 </span><span class="lineCov"> 459000 : asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");</span></a>
|
|
<a name="230"><span class="lineNum"> 230 </span><span class="lineCov"> 459000 : asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");</span></a>
|
|
<a name="231"><span class="lineNum"> 231 </span><span class="lineCov"> 459000 : asm volatile ("vpand %ymm7,%ymm4,%ymm4");</span></a>
|
|
<a name="232"><span class="lineNum"> 232 </span><span class="lineCov"> 459000 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="233"><span class="lineNum"> 233 </span><span class="lineCov"> 459000 : asm volatile ("vpxor %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="234"><span class="lineNum"> 234 </span><span class="lineCov"> 459000 : asm volatile ("vpxor %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="235"><span class="lineNum"> 235 </span> : </a>
|
|
<a name="236"><span class="lineNum"> 236 </span><span class="lineCov"> 459000 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="237"><span class="lineNum"> 237 </span><span class="lineCov"> 459000 : asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32]));</span></a>
|
|
<a name="238"><span class="lineNum"> 238 </span><span class="lineCov"> 459000 : asm volatile ("vpxor %ymm4,%ymm0,%ymm0");</span></a>
|
|
<a name="239"><span class="lineNum"> 239 </span><span class="lineCov"> 459000 : asm volatile ("vpxor %ymm5,%ymm1,%ymm1");</span></a>
|
|
<a name="240"><span class="lineNum"> 240 </span><span class="lineCov"> 459000 : asm volatile ("vpxor %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="241"><span class="lineNum"> 241 </span><span class="lineCov"> 459000 : asm volatile ("vpxor %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="242"><span class="lineNum"> 242 </span> : }</a>
|
|
<a name="243"><span class="lineNum"> 243 </span><span class="lineCov"> 65548 : asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="244"><span class="lineNum"> 244 </span><span class="lineCov"> 65548 : asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));</span></a>
|
|
<a name="245"><span class="lineNum"> 245 </span><span class="lineCov"> 65548 : asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i]));</span></a>
|
|
<a name="246"><span class="lineNum"> 246 </span><span class="lineCov"> 65548 : asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));</span></a>
|
|
<a name="247"><span class="lineNum"> 247 </span> : }</a>
|
|
<a name="248"><span class="lineNum"> 248 </span> : </a>
|
|
<a name="249"><span class="lineNum"> 249 </span> : raid_avx_end();</a>
|
|
<a name="250"><span class="lineNum"> 250 </span><span class="lineCov"> 19 : }</span></a>
|
|
<a name="251"><span class="lineNum"> 251 </span> : #endif</a>
|
|
<a name="252"><span class="lineNum"> 252 </span> : </a>
|
|
<a name="253"><span class="lineNum"> 253 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="254"><span class="lineNum"> 254 </span> : /*</a>
|
|
<a name="255"><span class="lineNum"> 255 </span> : * GEN2 (RAID6 with powers of 2) AVX512BW implementation</a>
|
|
<a name="256"><span class="lineNum"> 256 </span> : */</a>
|
|
<a name="257"><span class="lineNum"> 257 </span><span class="lineCov"> 654 : void raid_gen2_avx512bw(int nd, size_t size, void **vv)</span></a>
|
|
<a name="258"><span class="lineNum"> 258 </span> : {</a>
|
|
<a name="259"><span class="lineNum"> 259 </span><span class="lineCov"> 654 : uint8_t **v = (uint8_t**)vv;</span></a>
|
|
<a name="260"><span class="lineNum"> 260 </span> : uint8_t *p;</a>
|
|
<a name="261"><span class="lineNum"> 261 </span> : uint8_t *q;</a>
|
|
<a name="262"><span class="lineNum"> 262 </span> : int d, l;</a>
|
|
<a name="263"><span class="lineNum"> 263 </span> : size_t i;</a>
|
|
<a name="264"><span class="lineNum"> 264 </span> : </a>
|
|
<a name="265"><span class="lineNum"> 265 </span><span class="lineCov"> 654 : l = nd - 1;</span></a>
|
|
<a name="266"><span class="lineNum"> 266 </span><span class="lineCov"> 654 : p = v[nd];</span></a>
|
|
<a name="267"><span class="lineNum"> 267 </span><span class="lineCov"> 654 : q = v[nd + 1];</span></a>
|
|
<a name="268"><span class="lineNum"> 268 </span> : </a>
|
|
<a name="269"><span class="lineNum"> 269 </span> : raid_avx_begin();</a>
|
|
<a name="270"><span class="lineNum"> 270 </span> : </a>
|
|
<a name="271"><span class="lineNum"> 271 </span><span class="lineCov"> 654 : asm volatile ("vbroadcasti32x4 %0, %%zmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="272"><span class="lineNum"> 272 </span><span class="lineCov"> 654 : asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="273"><span class="lineNum"> 273 </span> : </a>
|
|
<a name="274"><span class="lineNum"> 274 </span><span class="lineCov"> 65370 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="275"><span class="lineNum"> 275 </span><span class="lineCov"> 64716 : asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="276"><span class="lineNum"> 276 </span><span class="lineCov"> 64716 : asm volatile ("vmovdqa64 %zmm0,%zmm1");</span></a>
|
|
<a name="277"><span class="lineNum"> 277 </span><span class="lineCov"> 534020 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="278"><span class="lineNum"> 278 </span><span class="lineCov"> 469304 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="279"><span class="lineNum"> 279 </span><span class="lineCov"> 469304 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="280"><span class="lineNum"> 280 </span><span class="lineCov"> 469304 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="281"><span class="lineNum"> 281 </span><span class="lineCov"> 469304 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="282"><span class="lineNum"> 282 </span><span class="lineCov"> 469304 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="283"><span class="lineNum"> 283 </span> : </a>
|
|
<a name="284"><span class="lineNum"> 284 </span><span class="lineCov"> 469304 : asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="285"><span class="lineNum"> 285 </span><span class="lineCov"> 469304 : asm volatile ("vpxorq %zmm4,%zmm0,%zmm0");</span></a>
|
|
<a name="286"><span class="lineNum"> 286 </span><span class="lineCov"> 469304 : asm volatile ("vpxorq %zmm4,%zmm1,%zmm1");</span></a>
|
|
<a name="287"><span class="lineNum"> 287 </span> : }</a>
|
|
<a name="288"><span class="lineNum"> 288 </span><span class="lineCov"> 64716 : asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="289"><span class="lineNum"> 289 </span><span class="lineCov"> 64716 : asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="290"><span class="lineNum"> 290 </span> : }</a>
|
|
<a name="291"><span class="lineNum"> 291 </span> : </a>
|
|
<a name="292"><span class="lineNum"> 292 </span> : raid_avx_end();</a>
|
|
<a name="293"><span class="lineNum"> 293 </span><span class="lineCov"> 654 : }</span></a>
|
|
<a name="294"><span class="lineNum"> 294 </span> : #endif</a>
|
|
<a name="295"><span class="lineNum"> 295 </span> : </a>
|
|
<a name="296"><span class="lineNum"> 296 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_SSE2)</a>
|
|
<a name="297"><span class="lineNum"> 297 </span> : /*</a>
|
|
<a name="298"><span class="lineNum"> 298 </span> : * GEN2 (RAID6 with powers of 2) SSE2 implementation</a>
|
|
<a name="299"><span class="lineNum"> 299 </span> : *</a>
|
|
<a name="300"><span class="lineNum"> 300 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="301"><span class="lineNum"> 301 </span> : */</a>
|
|
<a name="302"><span class="lineNum"> 302 </span><span class="lineCov"> 269102 : void raid_gen2_sse2ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="303"><span class="lineNum"> 303 </span> : {</a>
|
|
<a name="304"><span class="lineNum"> 304 </span><span class="lineCov"> 269102 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="305"><span class="lineNum"> 305 </span> : uint8_t *p;</a>
|
|
<a name="306"><span class="lineNum"> 306 </span> : uint8_t *q;</a>
|
|
<a name="307"><span class="lineNum"> 307 </span> : int d, l;</a>
|
|
<a name="308"><span class="lineNum"> 308 </span> : size_t i;</a>
|
|
<a name="309"><span class="lineNum"> 309 </span> : </a>
|
|
<a name="310"><span class="lineNum"> 310 </span><span class="lineCov"> 269102 : l = nd - 1;</span></a>
|
|
<a name="311"><span class="lineNum"> 311 </span><span class="lineCov"> 269102 : p = v[nd];</span></a>
|
|
<a name="312"><span class="lineNum"> 312 </span><span class="lineCov"> 269102 : q = v[nd + 1];</span></a>
|
|
<a name="313"><span class="lineNum"> 313 </span> : </a>
|
|
<a name="314"><span class="lineNum"> 314 </span> : raid_sse_begin();</a>
|
|
<a name="315"><span class="lineNum"> 315 </span> : </a>
|
|
<a name="316"><span class="lineNum"> 316 </span><span class="lineCov"> 269102 : asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="317"><span class="lineNum"> 317 </span> : </a>
|
|
<a name="318"><span class="lineNum"> 318 </span><span class="lineCov"> 4778698 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="319"><span class="lineNum"> 319 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));</span></a>
|
|
<a name="320"><span class="lineNum"> 320 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));</span></a>
|
|
<a name="321"><span class="lineNum"> 321 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %0,%%xmm2" : : "m" (v[l][i + 32]));</span></a>
|
|
<a name="322"><span class="lineNum"> 322 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %0,%%xmm3" : : "m" (v[l][i + 48]));</span></a>
|
|
<a name="323"><span class="lineNum"> 323 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %xmm0,%xmm4");</span></a>
|
|
<a name="324"><span class="lineNum"> 324 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %xmm1,%xmm5");</span></a>
|
|
<a name="325"><span class="lineNum"> 325 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %xmm2,%xmm6");</span></a>
|
|
<a name="326"><span class="lineNum"> 326 </span><span class="lineCov"> 4509596 : asm volatile ("movdqa %xmm3,%xmm7");</span></a>
|
|
<a name="327"><span class="lineNum"> 327 </span><span class="lineCov"> 27467364 : for (d = l - 1; d >= 0; --d) {</span></a>
|
|
<a name="328"><span class="lineNum"> 328 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm8,%xmm8");</span></a>
|
|
<a name="329"><span class="lineNum"> 329 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm9,%xmm9");</span></a>
|
|
<a name="330"><span class="lineNum"> 330 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm10,%xmm10");</span></a>
|
|
<a name="331"><span class="lineNum"> 331 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm11,%xmm11");</span></a>
|
|
<a name="332"><span class="lineNum"> 332 </span><span class="lineCov"> 22957768 : asm volatile ("pcmpgtb %xmm4,%xmm8");</span></a>
|
|
<a name="333"><span class="lineNum"> 333 </span><span class="lineCov"> 22957768 : asm volatile ("pcmpgtb %xmm5,%xmm9");</span></a>
|
|
<a name="334"><span class="lineNum"> 334 </span><span class="lineCov"> 22957768 : asm volatile ("pcmpgtb %xmm6,%xmm10");</span></a>
|
|
<a name="335"><span class="lineNum"> 335 </span><span class="lineCov"> 22957768 : asm volatile ("pcmpgtb %xmm7,%xmm11");</span></a>
|
|
<a name="336"><span class="lineNum"> 336 </span><span class="lineCov"> 22957768 : asm volatile ("paddb %xmm4,%xmm4");</span></a>
|
|
<a name="337"><span class="lineNum"> 337 </span><span class="lineCov"> 22957768 : asm volatile ("paddb %xmm5,%xmm5");</span></a>
|
|
<a name="338"><span class="lineNum"> 338 </span><span class="lineCov"> 22957768 : asm volatile ("paddb %xmm6,%xmm6");</span></a>
|
|
<a name="339"><span class="lineNum"> 339 </span><span class="lineCov"> 22957768 : asm volatile ("paddb %xmm7,%xmm7");</span></a>
|
|
<a name="340"><span class="lineNum"> 340 </span><span class="lineCov"> 22957768 : asm volatile ("pand %xmm15,%xmm8");</span></a>
|
|
<a name="341"><span class="lineNum"> 341 </span><span class="lineCov"> 22957768 : asm volatile ("pand %xmm15,%xmm9");</span></a>
|
|
<a name="342"><span class="lineNum"> 342 </span><span class="lineCov"> 22957768 : asm volatile ("pand %xmm15,%xmm10");</span></a>
|
|
<a name="343"><span class="lineNum"> 343 </span><span class="lineCov"> 22957768 : asm volatile ("pand %xmm15,%xmm11");</span></a>
|
|
<a name="344"><span class="lineNum"> 344 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm8,%xmm4");</span></a>
|
|
<a name="345"><span class="lineNum"> 345 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm9,%xmm5");</span></a>
|
|
<a name="346"><span class="lineNum"> 346 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm10,%xmm6");</span></a>
|
|
<a name="347"><span class="lineNum"> 347 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm11,%xmm7");</span></a>
|
|
<a name="348"><span class="lineNum"> 348 </span> : </a>
|
|
<a name="349"><span class="lineNum"> 349 </span><span class="lineCov"> 22957768 : asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));</span></a>
|
|
<a name="350"><span class="lineNum"> 350 </span><span class="lineCov"> 22957768 : asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));</span></a>
|
|
<a name="351"><span class="lineNum"> 351 </span><span class="lineCov"> 22957768 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i + 32]));</span></a>
|
|
<a name="352"><span class="lineNum"> 352 </span><span class="lineCov"> 22957768 : asm volatile ("movdqa %0,%%xmm11" : : "m" (v[d][i + 48]));</span></a>
|
|
<a name="353"><span class="lineNum"> 353 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm8,%xmm0");</span></a>
|
|
<a name="354"><span class="lineNum"> 354 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm9,%xmm1");</span></a>
|
|
<a name="355"><span class="lineNum"> 355 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm10,%xmm2");</span></a>
|
|
<a name="356"><span class="lineNum"> 356 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm11,%xmm3");</span></a>
|
|
<a name="357"><span class="lineNum"> 357 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm8,%xmm4");</span></a>
|
|
<a name="358"><span class="lineNum"> 358 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm9,%xmm5");</span></a>
|
|
<a name="359"><span class="lineNum"> 359 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm10,%xmm6");</span></a>
|
|
<a name="360"><span class="lineNum"> 360 </span><span class="lineCov"> 22957768 : asm volatile ("pxor %xmm11,%xmm7");</span></a>
|
|
<a name="361"><span class="lineNum"> 361 </span> : }</a>
|
|
<a name="362"><span class="lineNum"> 362 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="363"><span class="lineNum"> 363 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm1,%0" : "=m" (p[i + 16]));</span></a>
|
|
<a name="364"><span class="lineNum"> 364 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm2,%0" : "=m" (p[i + 32]));</span></a>
|
|
<a name="365"><span class="lineNum"> 365 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48]));</span></a>
|
|
<a name="366"><span class="lineNum"> 366 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm4,%0" : "=m" (q[i]));</span></a>
|
|
<a name="367"><span class="lineNum"> 367 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm5,%0" : "=m" (q[i + 16]));</span></a>
|
|
<a name="368"><span class="lineNum"> 368 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i + 32]));</span></a>
|
|
<a name="369"><span class="lineNum"> 369 </span><span class="lineCov"> 4509596 : asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48]));</span></a>
|
|
<a name="370"><span class="lineNum"> 370 </span> : }</a>
|
|
<a name="371"><span class="lineNum"> 371 </span> : </a>
|
|
<a name="372"><span class="lineNum"> 372 </span> : raid_sse_end();</a>
|
|
<a name="373"><span class="lineNum"> 373 </span><span class="lineCov"> 269102 : }</span></a>
|
|
<a name="374"><span class="lineNum"> 374 </span> : #endif</a>
|
|
<a name="375"><span class="lineNum"> 375 </span> : </a>
|
|
<a name="376"><span class="lineNum"> 376 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="377"><span class="lineNum"> 377 </span> : /*</a>
|
|
<a name="378"><span class="lineNum"> 378 </span> : * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="379"><span class="lineNum"> 379 </span> : */</a>
|
|
<a name="380"><span class="lineNum"> 380 </span><span class="lineCov"> 14 : void raid_gen3_ssse3(int nd, size_t size, void **vv)</span></a>
|
|
<a name="381"><span class="lineNum"> 381 </span> : {</a>
|
|
<a name="382"><span class="lineNum"> 382 </span><span class="lineCov"> 14 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="383"><span class="lineNum"> 383 </span> : uint8_t *p;</a>
|
|
<a name="384"><span class="lineNum"> 384 </span> : uint8_t *q;</a>
|
|
<a name="385"><span class="lineNum"> 385 </span> : uint8_t *r;</a>
|
|
<a name="386"><span class="lineNum"> 386 </span> : int d, l;</a>
|
|
<a name="387"><span class="lineNum"> 387 </span> : size_t i;</a>
|
|
<a name="388"><span class="lineNum"> 388 </span> : </a>
|
|
<a name="389"><span class="lineNum"> 389 </span><span class="lineCov"> 14 : l = nd - 1;</span></a>
|
|
<a name="390"><span class="lineNum"> 390 </span><span class="lineCov"> 14 : p = v[nd];</span></a>
|
|
<a name="391"><span class="lineNum"> 391 </span><span class="lineCov"> 14 : q = v[nd + 1];</span></a>
|
|
<a name="392"><span class="lineNum"> 392 </span><span class="lineCov"> 14 : r = v[nd + 2];</span></a>
|
|
<a name="393"><span class="lineNum"> 393 </span> : </a>
|
|
<a name="394"><span class="lineNum"> 394 </span> : /* special case with only one data disk */</a>
|
|
<a name="395"><span class="lineNum"> 395 </span><span class="lineCov"> 14 : if (l == 0) {</span></a>
|
|
<a name="396"><span class="lineNum"> 396 </span><span class="lineCov"> 4 : for (i = 0; i < 3; ++i)</span></a>
|
|
<a name="397"><span class="lineNum"> 397 </span><span class="lineCov"> 3 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="398"><span class="lineNum"> 398 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="399"><span class="lineNum"> 399 </span> : }</a>
|
|
<a name="400"><span class="lineNum"> 400 </span> : </a>
|
|
<a name="401"><span class="lineNum"> 401 </span> : raid_sse_begin();</a>
|
|
<a name="402"><span class="lineNum"> 402 </span> : </a>
|
|
<a name="403"><span class="lineNum"> 403 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="404"><span class="lineNum"> 404 </span><span class="lineCov"> 13 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="405"><span class="lineNum"> 405 </span><span class="lineCov"> 13 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="406"><span class="lineNum"> 406 </span> : </a>
|
|
<a name="407"><span class="lineNum"> 407 </span><span class="lineCov"> 196637 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="408"><span class="lineNum"> 408 </span> : /* last disk without the multiplication by two */</a>
|
|
<a name="409"><span class="lineNum"> 409 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="410"><span class="lineNum"> 410 </span> : </a>
|
|
<a name="411"><span class="lineNum"> 411 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %xmm4,%xmm0");</span></a>
|
|
<a name="412"><span class="lineNum"> 412 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %xmm4,%xmm1");</span></a>
|
|
<a name="413"><span class="lineNum"> 413 </span> : </a>
|
|
<a name="414"><span class="lineNum"> 414 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="415"><span class="lineNum"> 415 </span><span class="lineCov"> 196624 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="416"><span class="lineNum"> 416 </span><span class="lineCov"> 196624 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="417"><span class="lineNum"> 417 </span><span class="lineCov"> 196624 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="418"><span class="lineNum"> 418 </span> : </a>
|
|
<a name="419"><span class="lineNum"> 419 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="420"><span class="lineNum"> 420 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="421"><span class="lineNum"> 421 </span><span class="lineCov"> 196624 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="422"><span class="lineNum"> 422 </span><span class="lineCov"> 196624 : asm volatile ("pshufb %xmm5,%xmm6");</span></a>
|
|
<a name="423"><span class="lineNum"> 423 </span><span class="lineCov"> 196624 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="424"><span class="lineNum"> 424 </span> : </a>
|
|
<a name="425"><span class="lineNum"> 425 </span> : /* intermediate disks */</a>
|
|
<a name="426"><span class="lineNum"> 426 </span><span class="lineCov"> 1376752 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="427"><span class="lineNum"> 427 </span><span class="lineCov"> 1180128 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="428"><span class="lineNum"> 428 </span> : </a>
|
|
<a name="429"><span class="lineNum"> 429 </span><span class="lineCov"> 1180128 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="430"><span class="lineNum"> 430 </span><span class="lineCov"> 1180128 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="431"><span class="lineNum"> 431 </span><span class="lineCov"> 1180128 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="432"><span class="lineNum"> 432 </span><span class="lineCov"> 1180128 : asm volatile ("pand %xmm3,%xmm5");</span></a>
|
|
<a name="433"><span class="lineNum"> 433 </span><span class="lineCov"> 1180128 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="434"><span class="lineNum"> 434 </span> : </a>
|
|
<a name="435"><span class="lineNum"> 435 </span><span class="lineCov"> 1180128 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="436"><span class="lineNum"> 436 </span><span class="lineCov"> 1180128 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="437"><span class="lineNum"> 437 </span> : </a>
|
|
<a name="438"><span class="lineNum"> 438 </span><span class="lineCov"> 1180128 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="439"><span class="lineNum"> 439 </span><span class="lineCov"> 1180128 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="440"><span class="lineNum"> 440 </span><span class="lineCov"> 1180128 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="441"><span class="lineNum"> 441 </span><span class="lineCov"> 1180128 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="442"><span class="lineNum"> 442 </span> : </a>
|
|
<a name="443"><span class="lineNum"> 443 </span><span class="lineCov"> 1180128 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="444"><span class="lineNum"> 444 </span><span class="lineCov"> 1180128 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="445"><span class="lineNum"> 445 </span><span class="lineCov"> 1180128 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="446"><span class="lineNum"> 446 </span><span class="lineCov"> 1180128 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="447"><span class="lineNum"> 447 </span><span class="lineCov"> 1180128 : asm volatile ("pshufb %xmm5,%xmm6");</span></a>
|
|
<a name="448"><span class="lineNum"> 448 </span><span class="lineCov"> 1180128 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="449"><span class="lineNum"> 449 </span> : }</a>
|
|
<a name="450"><span class="lineNum"> 450 </span> : </a>
|
|
<a name="451"><span class="lineNum"> 451 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="452"><span class="lineNum"> 452 </span><span class="lineCov"> 196624 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="453"><span class="lineNum"> 453 </span> : </a>
|
|
<a name="454"><span class="lineNum"> 454 </span><span class="lineCov"> 196624 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="455"><span class="lineNum"> 455 </span><span class="lineCov"> 196624 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="456"><span class="lineNum"> 456 </span><span class="lineCov"> 196624 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="457"><span class="lineNum"> 457 </span><span class="lineCov"> 196624 : asm volatile ("pand %xmm3,%xmm5");</span></a>
|
|
<a name="458"><span class="lineNum"> 458 </span><span class="lineCov"> 196624 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="459"><span class="lineNum"> 459 </span> : </a>
|
|
<a name="460"><span class="lineNum"> 460 </span><span class="lineCov"> 196624 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="461"><span class="lineNum"> 461 </span><span class="lineCov"> 196624 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="462"><span class="lineNum"> 462 </span><span class="lineCov"> 196624 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="463"><span class="lineNum"> 463 </span> : </a>
|
|
<a name="464"><span class="lineNum"> 464 </span><span class="lineCov"> 196624 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="465"><span class="lineNum"> 465 </span><span class="lineCov"> 196624 : asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="466"><span class="lineNum"> 466 </span><span class="lineCov"> 196624 : asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="467"><span class="lineNum"> 467 </span> : }</a>
|
|
<a name="468"><span class="lineNum"> 468 </span> : </a>
|
|
<a name="469"><span class="lineNum"> 469 </span> : raid_sse_end();</a>
|
|
<a name="470"><span class="lineNum"> 470 </span> : }</a>
|
|
<a name="471"><span class="lineNum"> 471 </span> : #endif</a>
|
|
<a name="472"><span class="lineNum"> 472 </span> : </a>
|
|
<a name="473"><span class="lineNum"> 473 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_SSSE3)</a>
|
|
<a name="474"><span class="lineNum"> 474 </span> : /*</a>
|
|
<a name="475"><span class="lineNum"> 475 </span> : * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="476"><span class="lineNum"> 476 </span> : *</a>
|
|
<a name="477"><span class="lineNum"> 477 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="478"><span class="lineNum"> 478 </span> : */</a>
|
|
<a name="479"><span class="lineNum"> 479 </span><span class="lineCov"> 28231 : void raid_gen3_ssse3ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="480"><span class="lineNum"> 480 </span> : {</a>
|
|
<a name="481"><span class="lineNum"> 481 </span><span class="lineCov"> 28231 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="482"><span class="lineNum"> 482 </span> : uint8_t *p;</a>
|
|
<a name="483"><span class="lineNum"> 483 </span> : uint8_t *q;</a>
|
|
<a name="484"><span class="lineNum"> 484 </span> : uint8_t *r;</a>
|
|
<a name="485"><span class="lineNum"> 485 </span> : int d, l;</a>
|
|
<a name="486"><span class="lineNum"> 486 </span> : size_t i;</a>
|
|
<a name="487"><span class="lineNum"> 487 </span> : </a>
|
|
<a name="488"><span class="lineNum"> 488 </span><span class="lineCov"> 28231 : l = nd - 1;</span></a>
|
|
<a name="489"><span class="lineNum"> 489 </span><span class="lineCov"> 28231 : p = v[nd];</span></a>
|
|
<a name="490"><span class="lineNum"> 490 </span><span class="lineCov"> 28231 : q = v[nd + 1];</span></a>
|
|
<a name="491"><span class="lineNum"> 491 </span><span class="lineCov"> 28231 : r = v[nd + 2];</span></a>
|
|
<a name="492"><span class="lineNum"> 492 </span> : </a>
|
|
<a name="493"><span class="lineNum"> 493 </span> : /* special case with only one data disk */</a>
|
|
<a name="494"><span class="lineNum"> 494 </span><span class="lineCov"> 28231 : if (l == 0) {</span></a>
|
|
<a name="495"><span class="lineNum"> 495 </span><span class="lineCov"> 4 : for (i = 0; i < 3; ++i)</span></a>
|
|
<a name="496"><span class="lineNum"> 496 </span><span class="lineCov"> 3 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="497"><span class="lineNum"> 497 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="498"><span class="lineNum"> 498 </span> : }</a>
|
|
<a name="499"><span class="lineNum"> 499 </span> : </a>
|
|
<a name="500"><span class="lineNum"> 500 </span> : raid_sse_begin();</a>
|
|
<a name="501"><span class="lineNum"> 501 </span> : </a>
|
|
<a name="502"><span class="lineNum"> 502 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="503"><span class="lineNum"> 503 </span><span class="lineCov"> 28230 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="504"><span class="lineNum"> 504 </span><span class="lineCov"> 28230 : asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="505"><span class="lineNum"> 505 </span> : </a>
|
|
<a name="506"><span class="lineNum"> 506 </span><span class="lineCov"> 1160046 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="507"><span class="lineNum"> 507 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="508"><span class="lineNum"> 508 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="509"><span class="lineNum"> 509 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));</span></a>
|
|
<a name="510"><span class="lineNum"> 510 </span> : </a>
|
|
<a name="511"><span class="lineNum"> 511 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm4,%xmm0");</span></a>
|
|
<a name="512"><span class="lineNum"> 512 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm4,%xmm1");</span></a>
|
|
<a name="513"><span class="lineNum"> 513 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm12,%xmm8");</span></a>
|
|
<a name="514"><span class="lineNum"> 514 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm12,%xmm9");</span></a>
|
|
<a name="515"><span class="lineNum"> 515 </span> : </a>
|
|
<a name="516"><span class="lineNum"> 516 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="517"><span class="lineNum"> 517 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm12,%xmm13");</span></a>
|
|
<a name="518"><span class="lineNum"> 518 </span><span class="lineCov"> 1131816 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="519"><span class="lineNum"> 519 </span><span class="lineCov"> 1131816 : asm volatile ("psrlw $4,%xmm13");</span></a>
|
|
<a name="520"><span class="lineNum"> 520 </span><span class="lineCov"> 1131816 : asm volatile ("pand %xmm11,%xmm4");</span></a>
|
|
<a name="521"><span class="lineNum"> 521 </span><span class="lineCov"> 1131816 : asm volatile ("pand %xmm11,%xmm12");</span></a>
|
|
<a name="522"><span class="lineNum"> 522 </span><span class="lineCov"> 1131816 : asm volatile ("pand %xmm11,%xmm5");</span></a>
|
|
<a name="523"><span class="lineNum"> 523 </span><span class="lineCov"> 1131816 : asm volatile ("pand %xmm11,%xmm13");</span></a>
|
|
<a name="524"><span class="lineNum"> 524 </span> : </a>
|
|
<a name="525"><span class="lineNum"> 525 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="526"><span class="lineNum"> 526 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="527"><span class="lineNum"> 527 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm2,%xmm10");</span></a>
|
|
<a name="528"><span class="lineNum"> 528 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %xmm7,%xmm15");</span></a>
|
|
<a name="529"><span class="lineNum"> 529 </span><span class="lineCov"> 1131816 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="530"><span class="lineNum"> 530 </span><span class="lineCov"> 1131816 : asm volatile ("pshufb %xmm12,%xmm10");</span></a>
|
|
<a name="531"><span class="lineNum"> 531 </span><span class="lineCov"> 1131816 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="532"><span class="lineNum"> 532 </span><span class="lineCov"> 1131816 : asm volatile ("pshufb %xmm13,%xmm15");</span></a>
|
|
<a name="533"><span class="lineNum"> 533 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="534"><span class="lineNum"> 534 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm15,%xmm10");</span></a>
|
|
<a name="535"><span class="lineNum"> 535 </span> : </a>
|
|
<a name="536"><span class="lineNum"> 536 </span> : /* intermediate disks */</a>
|
|
<a name="537"><span class="lineNum"> 537 </span><span class="lineCov"> 6118040 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="538"><span class="lineNum"> 538 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="539"><span class="lineNum"> 539 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));</span></a>
|
|
<a name="540"><span class="lineNum"> 540 </span> : </a>
|
|
<a name="541"><span class="lineNum"> 541 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="542"><span class="lineNum"> 542 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm13,%xmm13");</span></a>
|
|
<a name="543"><span class="lineNum"> 543 </span><span class="lineCov"> 4986224 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="544"><span class="lineNum"> 544 </span><span class="lineCov"> 4986224 : asm volatile ("pcmpgtb %xmm9,%xmm13");</span></a>
|
|
<a name="545"><span class="lineNum"> 545 </span><span class="lineCov"> 4986224 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="546"><span class="lineNum"> 546 </span><span class="lineCov"> 4986224 : asm volatile ("paddb %xmm9,%xmm9");</span></a>
|
|
<a name="547"><span class="lineNum"> 547 </span><span class="lineCov"> 4986224 : asm volatile ("pand %xmm3,%xmm5");</span></a>
|
|
<a name="548"><span class="lineNum"> 548 </span><span class="lineCov"> 4986224 : asm volatile ("pand %xmm3,%xmm13");</span></a>
|
|
<a name="549"><span class="lineNum"> 549 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="550"><span class="lineNum"> 550 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm13,%xmm9");</span></a>
|
|
<a name="551"><span class="lineNum"> 551 </span> : </a>
|
|
<a name="552"><span class="lineNum"> 552 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="553"><span class="lineNum"> 553 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="554"><span class="lineNum"> 554 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm12,%xmm8");</span></a>
|
|
<a name="555"><span class="lineNum"> 555 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm12,%xmm9");</span></a>
|
|
<a name="556"><span class="lineNum"> 556 </span> : </a>
|
|
<a name="557"><span class="lineNum"> 557 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="558"><span class="lineNum"> 558 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %xmm12,%xmm13");</span></a>
|
|
<a name="559"><span class="lineNum"> 559 </span><span class="lineCov"> 4986224 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="560"><span class="lineNum"> 560 </span><span class="lineCov"> 4986224 : asm volatile ("psrlw $4,%xmm13");</span></a>
|
|
<a name="561"><span class="lineNum"> 561 </span><span class="lineCov"> 4986224 : asm volatile ("pand %xmm11,%xmm4");</span></a>
|
|
<a name="562"><span class="lineNum"> 562 </span><span class="lineCov"> 4986224 : asm volatile ("pand %xmm11,%xmm12");</span></a>
|
|
<a name="563"><span class="lineNum"> 563 </span><span class="lineCov"> 4986224 : asm volatile ("pand %xmm11,%xmm5");</span></a>
|
|
<a name="564"><span class="lineNum"> 564 </span><span class="lineCov"> 4986224 : asm volatile ("pand %xmm11,%xmm13");</span></a>
|
|
<a name="565"><span class="lineNum"> 565 </span> : </a>
|
|
<a name="566"><span class="lineNum"> 566 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="567"><span class="lineNum"> 567 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="568"><span class="lineNum"> 568 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %xmm6,%xmm14");</span></a>
|
|
<a name="569"><span class="lineNum"> 569 </span><span class="lineCov"> 4986224 : asm volatile ("movdqa %xmm7,%xmm15");</span></a>
|
|
<a name="570"><span class="lineNum"> 570 </span><span class="lineCov"> 4986224 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="571"><span class="lineNum"> 571 </span><span class="lineCov"> 4986224 : asm volatile ("pshufb %xmm12,%xmm14");</span></a>
|
|
<a name="572"><span class="lineNum"> 572 </span><span class="lineCov"> 4986224 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="573"><span class="lineNum"> 573 </span><span class="lineCov"> 4986224 : asm volatile ("pshufb %xmm13,%xmm15");</span></a>
|
|
<a name="574"><span class="lineNum"> 574 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="575"><span class="lineNum"> 575 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm14,%xmm10");</span></a>
|
|
<a name="576"><span class="lineNum"> 576 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="577"><span class="lineNum"> 577 </span><span class="lineCov"> 4986224 : asm volatile ("pxor %xmm15,%xmm10");</span></a>
|
|
<a name="578"><span class="lineNum"> 578 </span> : }</a>
|
|
<a name="579"><span class="lineNum"> 579 </span> : </a>
|
|
<a name="580"><span class="lineNum"> 580 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="581"><span class="lineNum"> 581 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="582"><span class="lineNum"> 582 </span><span class="lineCov"> 1131816 : asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));</span></a>
|
|
<a name="583"><span class="lineNum"> 583 </span> : </a>
|
|
<a name="584"><span class="lineNum"> 584 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="585"><span class="lineNum"> 585 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm13,%xmm13");</span></a>
|
|
<a name="586"><span class="lineNum"> 586 </span><span class="lineCov"> 1131816 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="587"><span class="lineNum"> 587 </span><span class="lineCov"> 1131816 : asm volatile ("pcmpgtb %xmm9,%xmm13");</span></a>
|
|
<a name="588"><span class="lineNum"> 588 </span><span class="lineCov"> 1131816 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="589"><span class="lineNum"> 589 </span><span class="lineCov"> 1131816 : asm volatile ("paddb %xmm9,%xmm9");</span></a>
|
|
<a name="590"><span class="lineNum"> 590 </span><span class="lineCov"> 1131816 : asm volatile ("pand %xmm3,%xmm5");</span></a>
|
|
<a name="591"><span class="lineNum"> 591 </span><span class="lineCov"> 1131816 : asm volatile ("pand %xmm3,%xmm13");</span></a>
|
|
<a name="592"><span class="lineNum"> 592 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="593"><span class="lineNum"> 593 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm13,%xmm9");</span></a>
|
|
<a name="594"><span class="lineNum"> 594 </span> : </a>
|
|
<a name="595"><span class="lineNum"> 595 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="596"><span class="lineNum"> 596 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="597"><span class="lineNum"> 597 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="598"><span class="lineNum"> 598 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm12,%xmm8");</span></a>
|
|
<a name="599"><span class="lineNum"> 599 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm12,%xmm9");</span></a>
|
|
<a name="600"><span class="lineNum"> 600 </span><span class="lineCov"> 1131816 : asm volatile ("pxor %xmm12,%xmm10");</span></a>
|
|
<a name="601"><span class="lineNum"> 601 </span> : </a>
|
|
<a name="602"><span class="lineNum"> 602 </span><span class="lineCov"> 1131816 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="603"><span class="lineNum"> 603 </span><span class="lineCov"> 1131816 : asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16]));</span></a>
|
|
<a name="604"><span class="lineNum"> 604 </span><span class="lineCov"> 1131816 : asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="605"><span class="lineNum"> 605 </span><span class="lineCov"> 1131816 : asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16]));</span></a>
|
|
<a name="606"><span class="lineNum"> 606 </span><span class="lineCov"> 1131816 : asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="607"><span class="lineNum"> 607 </span><span class="lineCov"> 1131816 : asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));</span></a>
|
|
<a name="608"><span class="lineNum"> 608 </span> : }</a>
|
|
<a name="609"><span class="lineNum"> 609 </span> : </a>
|
|
<a name="610"><span class="lineNum"> 610 </span> : raid_sse_end();</a>
|
|
<a name="611"><span class="lineNum"> 611 </span> : }</a>
|
|
<a name="612"><span class="lineNum"> 612 </span> : #endif</a>
|
|
<a name="613"><span class="lineNum"> 613 </span> : </a>
|
|
<a name="614"><span class="lineNum"> 614 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX2)</a>
|
|
<a name="615"><span class="lineNum"> 615 </span> : /*</a>
|
|
<a name="616"><span class="lineNum"> 616 </span> : * GEN3 (triple parity with Cauchy matrix) AVX2 implementation</a>
|
|
<a name="617"><span class="lineNum"> 617 </span> : *</a>
|
|
<a name="618"><span class="lineNum"> 618 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="619"><span class="lineNum"> 619 </span> : */</a>
|
|
<a name="620"><span class="lineNum"> 620 </span><span class="lineCov"> 12 : void raid_gen3_avx2ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="621"><span class="lineNum"> 621 </span> : {</a>
|
|
<a name="622"><span class="lineNum"> 622 </span><span class="lineCov"> 12 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="623"><span class="lineNum"> 623 </span> : uint8_t *p;</a>
|
|
<a name="624"><span class="lineNum"> 624 </span> : uint8_t *q;</a>
|
|
<a name="625"><span class="lineNum"> 625 </span> : uint8_t *r;</a>
|
|
<a name="626"><span class="lineNum"> 626 </span> : int d, l;</a>
|
|
<a name="627"><span class="lineNum"> 627 </span> : size_t i;</a>
|
|
<a name="628"><span class="lineNum"> 628 </span> : </a>
|
|
<a name="629"><span class="lineNum"> 629 </span><span class="lineCov"> 12 : l = nd - 1;</span></a>
|
|
<a name="630"><span class="lineNum"> 630 </span><span class="lineCov"> 12 : p = v[nd];</span></a>
|
|
<a name="631"><span class="lineNum"> 631 </span><span class="lineCov"> 12 : q = v[nd + 1];</span></a>
|
|
<a name="632"><span class="lineNum"> 632 </span><span class="lineCov"> 12 : r = v[nd + 2];</span></a>
|
|
<a name="633"><span class="lineNum"> 633 </span> : </a>
|
|
<a name="634"><span class="lineNum"> 634 </span> : /* special case with only one data disk */</a>
|
|
<a name="635"><span class="lineNum"> 635 </span><span class="lineCov"> 12 : if (l == 0) {</span></a>
|
|
<a name="636"><span class="lineNum"> 636 </span><span class="lineCov"> 4 : for (i = 0; i < 3; ++i)</span></a>
|
|
<a name="637"><span class="lineNum"> 637 </span><span class="lineCov"> 3 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="638"><span class="lineNum"> 638 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="639"><span class="lineNum"> 639 </span> : }</a>
|
|
<a name="640"><span class="lineNum"> 640 </span> : </a>
|
|
<a name="641"><span class="lineNum"> 641 </span> : raid_avx_begin();</a>
|
|
<a name="642"><span class="lineNum"> 642 </span> : </a>
|
|
<a name="643"><span class="lineNum"> 643 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="644"><span class="lineNum"> 644 </span><span class="lineCov"> 11 : asm volatile ("vbroadcasti128 %0, %%ymm3" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="645"><span class="lineNum"> 645 </span><span class="lineCov"> 11 : asm volatile ("vbroadcasti128 %0, %%ymm11" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="646"><span class="lineNum"> 646 </span> : </a>
|
|
<a name="647"><span class="lineNum"> 647 </span><span class="lineCov"> 40975 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="648"><span class="lineNum"> 648 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="649"><span class="lineNum"> 649 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="650"><span class="lineNum"> 650 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));</span></a>
|
|
<a name="651"><span class="lineNum"> 651 </span> : </a>
|
|
<a name="652"><span class="lineNum"> 652 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %ymm4,%ymm0");</span></a>
|
|
<a name="653"><span class="lineNum"> 653 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %ymm4,%ymm1");</span></a>
|
|
<a name="654"><span class="lineNum"> 654 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %ymm12,%ymm8");</span></a>
|
|
<a name="655"><span class="lineNum"> 655 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %ymm12,%ymm9");</span></a>
|
|
<a name="656"><span class="lineNum"> 656 </span> : </a>
|
|
<a name="657"><span class="lineNum"> 657 </span><span class="lineCov"> 40964 : asm volatile ("vpsrlw $4,%ymm4,%ymm5");</span></a>
|
|
<a name="658"><span class="lineNum"> 658 </span><span class="lineCov"> 40964 : asm volatile ("vpsrlw $4,%ymm12,%ymm13");</span></a>
|
|
<a name="659"><span class="lineNum"> 659 </span><span class="lineCov"> 40964 : asm volatile ("vpand %ymm11,%ymm4,%ymm4");</span></a>
|
|
<a name="660"><span class="lineNum"> 660 </span><span class="lineCov"> 40964 : asm volatile ("vpand %ymm11,%ymm12,%ymm12");</span></a>
|
|
<a name="661"><span class="lineNum"> 661 </span><span class="lineCov"> 40964 : asm volatile ("vpand %ymm11,%ymm5,%ymm5");</span></a>
|
|
<a name="662"><span class="lineNum"> 662 </span><span class="lineCov"> 40964 : asm volatile ("vpand %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="663"><span class="lineNum"> 663 </span> : </a>
|
|
<a name="664"><span class="lineNum"> 664 </span><span class="lineCov"> 40964 : asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="665"><span class="lineNum"> 665 </span><span class="lineCov"> 40964 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="666"><span class="lineNum"> 666 </span><span class="lineCov"> 40964 : asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");</span></a>
|
|
<a name="667"><span class="lineNum"> 667 </span><span class="lineCov"> 40964 : asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");</span></a>
|
|
<a name="668"><span class="lineNum"> 668 </span><span class="lineCov"> 40964 : asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");</span></a>
|
|
<a name="669"><span class="lineNum"> 669 </span><span class="lineCov"> 40964 : asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");</span></a>
|
|
<a name="670"><span class="lineNum"> 670 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm7,%ymm2,%ymm2");</span></a>
|
|
<a name="671"><span class="lineNum"> 671 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="672"><span class="lineNum"> 672 </span> : </a>
|
|
<a name="673"><span class="lineNum"> 673 </span> : /* intermediate disks */</a>
|
|
<a name="674"><span class="lineNum"> 674 </span><span class="lineCov"> 286844 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="675"><span class="lineNum"> 675 </span><span class="lineCov"> 245880 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="676"><span class="lineNum"> 676 </span><span class="lineCov"> 245880 : asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));</span></a>
|
|
<a name="677"><span class="lineNum"> 677 </span> : </a>
|
|
<a name="678"><span class="lineNum"> 678 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm5,%ymm5,%ymm5");</span></a>
|
|
<a name="679"><span class="lineNum"> 679 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm13,%ymm13,%ymm13");</span></a>
|
|
<a name="680"><span class="lineNum"> 680 </span><span class="lineCov"> 245880 : asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");</span></a>
|
|
<a name="681"><span class="lineNum"> 681 </span><span class="lineCov"> 245880 : asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");</span></a>
|
|
<a name="682"><span class="lineNum"> 682 </span><span class="lineCov"> 245880 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="683"><span class="lineNum"> 683 </span><span class="lineCov"> 245880 : asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");</span></a>
|
|
<a name="684"><span class="lineNum"> 684 </span><span class="lineCov"> 245880 : asm volatile ("vpand %ymm3,%ymm5,%ymm5");</span></a>
|
|
<a name="685"><span class="lineNum"> 685 </span><span class="lineCov"> 245880 : asm volatile ("vpand %ymm3,%ymm13,%ymm13");</span></a>
|
|
<a name="686"><span class="lineNum"> 686 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm5,%ymm1,%ymm1");</span></a>
|
|
<a name="687"><span class="lineNum"> 687 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm13,%ymm9,%ymm9");</span></a>
|
|
<a name="688"><span class="lineNum"> 688 </span> : </a>
|
|
<a name="689"><span class="lineNum"> 689 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm4,%ymm0,%ymm0");</span></a>
|
|
<a name="690"><span class="lineNum"> 690 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm4,%ymm1,%ymm1");</span></a>
|
|
<a name="691"><span class="lineNum"> 691 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm12,%ymm8,%ymm8");</span></a>
|
|
<a name="692"><span class="lineNum"> 692 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm12,%ymm9,%ymm9");</span></a>
|
|
<a name="693"><span class="lineNum"> 693 </span> : </a>
|
|
<a name="694"><span class="lineNum"> 694 </span><span class="lineCov"> 245880 : asm volatile ("vpsrlw $4,%ymm4,%ymm5");</span></a>
|
|
<a name="695"><span class="lineNum"> 695 </span><span class="lineCov"> 245880 : asm volatile ("vpsrlw $4,%ymm12,%ymm13");</span></a>
|
|
<a name="696"><span class="lineNum"> 696 </span><span class="lineCov"> 245880 : asm volatile ("vpand %ymm11,%ymm4,%ymm4");</span></a>
|
|
<a name="697"><span class="lineNum"> 697 </span><span class="lineCov"> 245880 : asm volatile ("vpand %ymm11,%ymm12,%ymm12");</span></a>
|
|
<a name="698"><span class="lineNum"> 698 </span><span class="lineCov"> 245880 : asm volatile ("vpand %ymm11,%ymm5,%ymm5");</span></a>
|
|
<a name="699"><span class="lineNum"> 699 </span><span class="lineCov"> 245880 : asm volatile ("vpand %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="700"><span class="lineNum"> 700 </span> : </a>
|
|
<a name="701"><span class="lineNum"> 701 </span><span class="lineCov"> 245880 : asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="702"><span class="lineNum"> 702 </span><span class="lineCov"> 245880 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="703"><span class="lineNum"> 703 </span><span class="lineCov"> 245880 : asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");</span></a>
|
|
<a name="704"><span class="lineNum"> 704 </span><span class="lineCov"> 245880 : asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");</span></a>
|
|
<a name="705"><span class="lineNum"> 705 </span><span class="lineCov"> 245880 : asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");</span></a>
|
|
<a name="706"><span class="lineNum"> 706 </span><span class="lineCov"> 245880 : asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");</span></a>
|
|
<a name="707"><span class="lineNum"> 707 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm6,%ymm2,%ymm2");</span></a>
|
|
<a name="708"><span class="lineNum"> 708 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm14,%ymm10,%ymm10");</span></a>
|
|
<a name="709"><span class="lineNum"> 709 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm7,%ymm2,%ymm2");</span></a>
|
|
<a name="710"><span class="lineNum"> 710 </span><span class="lineCov"> 245880 : asm volatile ("vpxor %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="711"><span class="lineNum"> 711 </span> : }</a>
|
|
<a name="712"><span class="lineNum"> 712 </span> : </a>
|
|
<a name="713"><span class="lineNum"> 713 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="714"><span class="lineNum"> 714 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="715"><span class="lineNum"> 715 </span><span class="lineCov"> 40964 : asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));</span></a>
|
|
<a name="716"><span class="lineNum"> 716 </span> : </a>
|
|
<a name="717"><span class="lineNum"> 717 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm5,%ymm5,%ymm5");</span></a>
|
|
<a name="718"><span class="lineNum"> 718 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm13,%ymm13,%ymm13");</span></a>
|
|
<a name="719"><span class="lineNum"> 719 </span><span class="lineCov"> 40964 : asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");</span></a>
|
|
<a name="720"><span class="lineNum"> 720 </span><span class="lineCov"> 40964 : asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");</span></a>
|
|
<a name="721"><span class="lineNum"> 721 </span><span class="lineCov"> 40964 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="722"><span class="lineNum"> 722 </span><span class="lineCov"> 40964 : asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");</span></a>
|
|
<a name="723"><span class="lineNum"> 723 </span><span class="lineCov"> 40964 : asm volatile ("vpand %ymm3,%ymm5,%ymm5");</span></a>
|
|
<a name="724"><span class="lineNum"> 724 </span><span class="lineCov"> 40964 : asm volatile ("vpand %ymm3,%ymm13,%ymm13");</span></a>
|
|
<a name="725"><span class="lineNum"> 725 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm5,%ymm1,%ymm1");</span></a>
|
|
<a name="726"><span class="lineNum"> 726 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm13,%ymm9,%ymm9");</span></a>
|
|
<a name="727"><span class="lineNum"> 727 </span> : </a>
|
|
<a name="728"><span class="lineNum"> 728 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm4,%ymm0,%ymm0");</span></a>
|
|
<a name="729"><span class="lineNum"> 729 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm4,%ymm1,%ymm1");</span></a>
|
|
<a name="730"><span class="lineNum"> 730 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="731"><span class="lineNum"> 731 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm12,%ymm8,%ymm8");</span></a>
|
|
<a name="732"><span class="lineNum"> 732 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm12,%ymm9,%ymm9");</span></a>
|
|
<a name="733"><span class="lineNum"> 733 </span><span class="lineCov"> 40964 : asm volatile ("vpxor %ymm12,%ymm10,%ymm10");</span></a>
|
|
<a name="734"><span class="lineNum"> 734 </span> : </a>
|
|
<a name="735"><span class="lineNum"> 735 </span><span class="lineCov"> 40964 : asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="736"><span class="lineNum"> 736 </span><span class="lineCov"> 40964 : asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32]));</span></a>
|
|
<a name="737"><span class="lineNum"> 737 </span><span class="lineCov"> 40964 : asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="738"><span class="lineNum"> 738 </span><span class="lineCov"> 40964 : asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32]));</span></a>
|
|
<a name="739"><span class="lineNum"> 739 </span><span class="lineCov"> 40964 : asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="740"><span class="lineNum"> 740 </span><span class="lineCov"> 40964 : asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32]));</span></a>
|
|
<a name="741"><span class="lineNum"> 741 </span> : }</a>
|
|
<a name="742"><span class="lineNum"> 742 </span> : </a>
|
|
<a name="743"><span class="lineNum"> 743 </span> : raid_avx_end();</a>
|
|
<a name="744"><span class="lineNum"> 744 </span> : }</a>
|
|
<a name="745"><span class="lineNum"> 745 </span> : #endif</a>
|
|
<a name="746"><span class="lineNum"> 746 </span> : </a>
|
|
<a name="747"><span class="lineNum"> 747 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="748"><span class="lineNum"> 748 </span> : /*</a>
|
|
<a name="749"><span class="lineNum"> 749 </span> : * GEN3 (triple parity with Cauchy matrix) AVX512BW implementation</a>
|
|
<a name="750"><span class="lineNum"> 750 </span> : */</a>
|
|
<a name="751"><span class="lineNum"> 751 </span><span class="lineCov"> 1476 : void raid_gen3_avx512bw(int nd, size_t size, void **vv)</span></a>
|
|
<a name="752"><span class="lineNum"> 752 </span> : {</a>
|
|
<a name="753"><span class="lineNum"> 753 </span><span class="lineCov"> 1476 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="754"><span class="lineNum"> 754 </span> : uint8_t *p;</a>
|
|
<a name="755"><span class="lineNum"> 755 </span> : uint8_t *q;</a>
|
|
<a name="756"><span class="lineNum"> 756 </span> : uint8_t *r;</a>
|
|
<a name="757"><span class="lineNum"> 757 </span> : int d, l;</a>
|
|
<a name="758"><span class="lineNum"> 758 </span> : size_t i;</a>
|
|
<a name="759"><span class="lineNum"> 759 </span> : </a>
|
|
<a name="760"><span class="lineNum"> 760 </span><span class="lineCov"> 1476 : l = nd - 1;</span></a>
|
|
<a name="761"><span class="lineNum"> 761 </span><span class="lineCov"> 1476 : p = v[nd];</span></a>
|
|
<a name="762"><span class="lineNum"> 762 </span><span class="lineCov"> 1476 : q = v[nd + 1];</span></a>
|
|
<a name="763"><span class="lineNum"> 763 </span><span class="lineCov"> 1476 : r = v[nd + 2];</span></a>
|
|
<a name="764"><span class="lineNum"> 764 </span> : </a>
|
|
<a name="765"><span class="lineNum"> 765 </span> : /* special case with only one data disk */</a>
|
|
<a name="766"><span class="lineNum"> 766 </span><span class="lineCov"> 1476 : if (l == 0) {</span></a>
|
|
<a name="767"><span class="lineNum"> 767 </span><span class="lineCov"> 4 : for (i = 0; i < 3; ++i)</span></a>
|
|
<a name="768"><span class="lineNum"> 768 </span><span class="lineCov"> 3 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="769"><span class="lineNum"> 769 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="770"><span class="lineNum"> 770 </span> : }</a>
|
|
<a name="771"><span class="lineNum"> 771 </span> : </a>
|
|
<a name="772"><span class="lineNum"> 772 </span> : raid_avx_begin();</a>
|
|
<a name="773"><span class="lineNum"> 773 </span> : </a>
|
|
<a name="774"><span class="lineNum"> 774 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="775"><span class="lineNum"> 775 </span><span class="lineCov"> 1475 : asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="776"><span class="lineNum"> 776 </span><span class="lineCov"> 1475 : asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="777"><span class="lineNum"> 777 </span><span class="lineCov"> 1475 : asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="778"><span class="lineNum"> 778 </span> : </a>
|
|
<a name="779"><span class="lineNum"> 779 </span><span class="lineCov"> 40711 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="780"><span class="lineNum"> 780 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="781"><span class="lineNum"> 781 </span><span class="lineCov"> 39236 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="782"><span class="lineNum"> 782 </span> : </a>
|
|
<a name="783"><span class="lineNum"> 783 </span><span class="lineCov"> 39236 : asm volatile ("vmovdqa64 %zmm10,%zmm0");</span></a>
|
|
<a name="784"><span class="lineNum"> 784 </span><span class="lineCov"> 39236 : asm volatile ("vmovdqa64 %zmm10,%zmm1");</span></a>
|
|
<a name="785"><span class="lineNum"> 785 </span> : </a>
|
|
<a name="786"><span class="lineNum"> 786 </span><span class="lineCov"> 39236 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="787"><span class="lineNum"> 787 </span><span class="lineCov"> 39236 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="788"><span class="lineNum"> 788 </span><span class="lineCov"> 39236 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="789"><span class="lineNum"> 789 </span> : </a>
|
|
<a name="790"><span class="lineNum"> 790 </span><span class="lineCov"> 39236 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="791"><span class="lineNum"> 791 </span><span class="lineCov"> 39236 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="792"><span class="lineNum"> 792 </span><span class="lineCov"> 39236 : asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="793"><span class="lineNum"> 793 </span><span class="lineCov"> 39236 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="794"><span class="lineNum"> 794 </span><span class="lineCov"> 39236 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="795"><span class="lineNum"> 795 </span> : </a>
|
|
<a name="796"><span class="lineNum"> 796 </span> : /* intermediate disks */</a>
|
|
<a name="797"><span class="lineNum"> 797 </span><span class="lineCov"> 303164 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="798"><span class="lineNum"> 798 </span><span class="lineCov"> 263928 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="799"><span class="lineNum"> 799 </span> : </a>
|
|
<a name="800"><span class="lineNum"> 800 </span><span class="lineCov"> 263928 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="801"><span class="lineNum"> 801 </span><span class="lineCov"> 263928 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="802"><span class="lineNum"> 802 </span><span class="lineCov"> 263928 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="803"><span class="lineNum"> 803 </span><span class="lineCov"> 263928 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="804"><span class="lineNum"> 804 </span><span class="lineCov"> 263928 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="805"><span class="lineNum"> 805 </span> : </a>
|
|
<a name="806"><span class="lineNum"> 806 </span><span class="lineCov"> 263928 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="807"><span class="lineNum"> 807 </span><span class="lineCov"> 263928 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="808"><span class="lineNum"> 808 </span> : </a>
|
|
<a name="809"><span class="lineNum"> 809 </span><span class="lineCov"> 263928 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="810"><span class="lineNum"> 810 </span><span class="lineCov"> 263928 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="811"><span class="lineNum"> 811 </span><span class="lineCov"> 263928 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="812"><span class="lineNum"> 812 </span> : </a>
|
|
<a name="813"><span class="lineNum"> 813 </span><span class="lineCov"> 263928 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="814"><span class="lineNum"> 814 </span><span class="lineCov"> 263928 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="815"><span class="lineNum"> 815 </span><span class="lineCov"> 263928 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="816"><span class="lineNum"> 816 </span><span class="lineCov"> 263928 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="817"><span class="lineNum"> 817 </span><span class="lineCov"> 263928 : asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");</span></a>
|
|
<a name="818"><span class="lineNum"> 818 </span><span class="lineCov"> 263928 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="819"><span class="lineNum"> 819 </span> : }</a>
|
|
<a name="820"><span class="lineNum"> 820 </span> : </a>
|
|
<a name="821"><span class="lineNum"> 821 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="822"><span class="lineNum"> 822 </span><span class="lineCov"> 39236 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="823"><span class="lineNum"> 823 </span> : </a>
|
|
<a name="824"><span class="lineNum"> 824 </span><span class="lineCov"> 39236 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="825"><span class="lineNum"> 825 </span><span class="lineCov"> 39236 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="826"><span class="lineNum"> 826 </span><span class="lineCov"> 39236 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="827"><span class="lineNum"> 827 </span><span class="lineCov"> 39236 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="828"><span class="lineNum"> 828 </span><span class="lineCov"> 39236 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="829"><span class="lineNum"> 829 </span> : </a>
|
|
<a name="830"><span class="lineNum"> 830 </span><span class="lineCov"> 39236 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="831"><span class="lineNum"> 831 </span><span class="lineCov"> 39236 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="832"><span class="lineNum"> 832 </span><span class="lineCov"> 39236 : asm volatile ("vpxorq %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="833"><span class="lineNum"> 833 </span> : </a>
|
|
<a name="834"><span class="lineNum"> 834 </span><span class="lineCov"> 39236 : asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="835"><span class="lineNum"> 835 </span><span class="lineCov"> 39236 : asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="836"><span class="lineNum"> 836 </span><span class="lineCov"> 39236 : asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="837"><span class="lineNum"> 837 </span> : }</a>
|
|
<a name="838"><span class="lineNum"> 838 </span> : </a>
|
|
<a name="839"><span class="lineNum"> 839 </span> : raid_avx_end();</a>
|
|
<a name="840"><span class="lineNum"> 840 </span> : }</a>
|
|
<a name="841"><span class="lineNum"> 841 </span> : #endif</a>
|
|
<a name="842"><span class="lineNum"> 842 </span> : </a>
|
|
<a name="843"><span class="lineNum"> 843 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="844"><span class="lineNum"> 844 </span> : /*</a>
|
|
<a name="845"><span class="lineNum"> 845 </span> : * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="846"><span class="lineNum"> 846 </span> : */</a>
|
|
<a name="847"><span class="lineNum"> 847 </span><span class="lineCov"> 10 : void raid_gen4_ssse3(int nd, size_t size, void **vv)</span></a>
|
|
<a name="848"><span class="lineNum"> 848 </span> : {</a>
|
|
<a name="849"><span class="lineNum"> 849 </span><span class="lineCov"> 10 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="850"><span class="lineNum"> 850 </span> : uint8_t *p;</a>
|
|
<a name="851"><span class="lineNum"> 851 </span> : uint8_t *q;</a>
|
|
<a name="852"><span class="lineNum"> 852 </span> : uint8_t *r;</a>
|
|
<a name="853"><span class="lineNum"> 853 </span> : uint8_t *s;</a>
|
|
<a name="854"><span class="lineNum"> 854 </span> : int d, l;</a>
|
|
<a name="855"><span class="lineNum"> 855 </span> : size_t i;</a>
|
|
<a name="856"><span class="lineNum"> 856 </span> : </a>
|
|
<a name="857"><span class="lineNum"> 857 </span><span class="lineCov"> 10 : l = nd - 1;</span></a>
|
|
<a name="858"><span class="lineNum"> 858 </span><span class="lineCov"> 10 : p = v[nd];</span></a>
|
|
<a name="859"><span class="lineNum"> 859 </span><span class="lineCov"> 10 : q = v[nd + 1];</span></a>
|
|
<a name="860"><span class="lineNum"> 860 </span><span class="lineCov"> 10 : r = v[nd + 2];</span></a>
|
|
<a name="861"><span class="lineNum"> 861 </span><span class="lineCov"> 10 : s = v[nd + 3];</span></a>
|
|
<a name="862"><span class="lineNum"> 862 </span> : </a>
|
|
<a name="863"><span class="lineNum"> 863 </span> : /* special case with only one data disk */</a>
|
|
<a name="864"><span class="lineNum"> 864 </span><span class="lineCov"> 10 : if (l == 0) {</span></a>
|
|
<a name="865"><span class="lineNum"> 865 </span><span class="lineCov"> 5 : for (i = 0; i < 4; ++i)</span></a>
|
|
<a name="866"><span class="lineNum"> 866 </span><span class="lineCov"> 4 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="867"><span class="lineNum"> 867 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="868"><span class="lineNum"> 868 </span> : }</a>
|
|
<a name="869"><span class="lineNum"> 869 </span> : </a>
|
|
<a name="870"><span class="lineNum"> 870 </span> : raid_sse_begin();</a>
|
|
<a name="871"><span class="lineNum"> 871 </span> : </a>
|
|
<a name="872"><span class="lineNum"> 872 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="873"><span class="lineNum"> 873 </span><span class="lineCov"> 131097 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="874"><span class="lineNum"> 874 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="875"><span class="lineNum"> 875 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="876"><span class="lineNum"> 876 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="877"><span class="lineNum"> 877 </span> : </a>
|
|
<a name="878"><span class="lineNum"> 878 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %xmm4,%xmm0");</span></a>
|
|
<a name="879"><span class="lineNum"> 879 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %xmm4,%xmm1");</span></a>
|
|
<a name="880"><span class="lineNum"> 880 </span> : </a>
|
|
<a name="881"><span class="lineNum"> 881 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="882"><span class="lineNum"> 882 </span><span class="lineCov"> 131088 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="883"><span class="lineNum"> 883 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="884"><span class="lineNum"> 884 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="885"><span class="lineNum"> 885 </span> : </a>
|
|
<a name="886"><span class="lineNum"> 886 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="887"><span class="lineNum"> 887 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="888"><span class="lineNum"> 888 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="889"><span class="lineNum"> 889 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="890"><span class="lineNum"> 890 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="891"><span class="lineNum"> 891 </span> : </a>
|
|
<a name="892"><span class="lineNum"> 892 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="893"><span class="lineNum"> 893 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="894"><span class="lineNum"> 894 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm3");</span></a>
|
|
<a name="895"><span class="lineNum"> 895 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="896"><span class="lineNum"> 896 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="897"><span class="lineNum"> 897 </span> : </a>
|
|
<a name="898"><span class="lineNum"> 898 </span> : /* intermediate disks */</a>
|
|
<a name="899"><span class="lineNum"> 899 </span><span class="lineCov"> 918000 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="900"><span class="lineNum"> 900 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="901"><span class="lineNum"> 901 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="902"><span class="lineNum"> 902 </span> : </a>
|
|
<a name="903"><span class="lineNum"> 903 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="904"><span class="lineNum"> 904 </span><span class="lineCov"> 786912 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="905"><span class="lineNum"> 905 </span><span class="lineCov"> 786912 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="906"><span class="lineNum"> 906 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="907"><span class="lineNum"> 907 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="908"><span class="lineNum"> 908 </span> : </a>
|
|
<a name="909"><span class="lineNum"> 909 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="910"><span class="lineNum"> 910 </span> : </a>
|
|
<a name="911"><span class="lineNum"> 911 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="912"><span class="lineNum"> 912 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="913"><span class="lineNum"> 913 </span> : </a>
|
|
<a name="914"><span class="lineNum"> 914 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="915"><span class="lineNum"> 915 </span><span class="lineCov"> 786912 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="916"><span class="lineNum"> 916 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="917"><span class="lineNum"> 917 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="918"><span class="lineNum"> 918 </span> : </a>
|
|
<a name="919"><span class="lineNum"> 919 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="920"><span class="lineNum"> 920 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="921"><span class="lineNum"> 921 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="922"><span class="lineNum"> 922 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="923"><span class="lineNum"> 923 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="924"><span class="lineNum"> 924 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="925"><span class="lineNum"> 925 </span> : </a>
|
|
<a name="926"><span class="lineNum"> 926 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="927"><span class="lineNum"> 927 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="928"><span class="lineNum"> 928 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="929"><span class="lineNum"> 929 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="930"><span class="lineNum"> 930 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm3");</span></a>
|
|
<a name="931"><span class="lineNum"> 931 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="932"><span class="lineNum"> 932 </span> : }</a>
|
|
<a name="933"><span class="lineNum"> 933 </span> : </a>
|
|
<a name="934"><span class="lineNum"> 934 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="935"><span class="lineNum"> 935 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="936"><span class="lineNum"> 936 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="937"><span class="lineNum"> 937 </span> : </a>
|
|
<a name="938"><span class="lineNum"> 938 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="939"><span class="lineNum"> 939 </span><span class="lineCov"> 131088 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="940"><span class="lineNum"> 940 </span><span class="lineCov"> 131088 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="941"><span class="lineNum"> 941 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="942"><span class="lineNum"> 942 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="943"><span class="lineNum"> 943 </span> : </a>
|
|
<a name="944"><span class="lineNum"> 944 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="945"><span class="lineNum"> 945 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="946"><span class="lineNum"> 946 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="947"><span class="lineNum"> 947 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm3");</span></a>
|
|
<a name="948"><span class="lineNum"> 948 </span> : </a>
|
|
<a name="949"><span class="lineNum"> 949 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="950"><span class="lineNum"> 950 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="951"><span class="lineNum"> 951 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="952"><span class="lineNum"> 952 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="953"><span class="lineNum"> 953 </span> : }</a>
|
|
<a name="954"><span class="lineNum"> 954 </span> : </a>
|
|
<a name="955"><span class="lineNum"> 955 </span> : raid_sse_end();</a>
|
|
<a name="956"><span class="lineNum"> 956 </span> : }</a>
|
|
<a name="957"><span class="lineNum"> 957 </span> : #endif</a>
|
|
<a name="958"><span class="lineNum"> 958 </span> : </a>
|
|
<a name="959"><span class="lineNum"> 959 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_SSSE3)</a>
|
|
<a name="960"><span class="lineNum"> 960 </span> : /*</a>
|
|
<a name="961"><span class="lineNum"> 961 </span> : * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="962"><span class="lineNum"> 962 </span> : *</a>
|
|
<a name="963"><span class="lineNum"> 963 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="964"><span class="lineNum"> 964 </span> : */</a>
|
|
<a name="965"><span class="lineNum"> 965 </span><span class="lineCov"> 28342 : void raid_gen4_ssse3ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="966"><span class="lineNum"> 966 </span> : {</a>
|
|
<a name="967"><span class="lineNum"> 967 </span><span class="lineCov"> 28342 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="968"><span class="lineNum"> 968 </span> : uint8_t *p;</a>
|
|
<a name="969"><span class="lineNum"> 969 </span> : uint8_t *q;</a>
|
|
<a name="970"><span class="lineNum"> 970 </span> : uint8_t *r;</a>
|
|
<a name="971"><span class="lineNum"> 971 </span> : uint8_t *s;</a>
|
|
<a name="972"><span class="lineNum"> 972 </span> : int d, l;</a>
|
|
<a name="973"><span class="lineNum"> 973 </span> : size_t i;</a>
|
|
<a name="974"><span class="lineNum"> 974 </span> : </a>
|
|
<a name="975"><span class="lineNum"> 975 </span><span class="lineCov"> 28342 : l = nd - 1;</span></a>
|
|
<a name="976"><span class="lineNum"> 976 </span><span class="lineCov"> 28342 : p = v[nd];</span></a>
|
|
<a name="977"><span class="lineNum"> 977 </span><span class="lineCov"> 28342 : q = v[nd + 1];</span></a>
|
|
<a name="978"><span class="lineNum"> 978 </span><span class="lineCov"> 28342 : r = v[nd + 2];</span></a>
|
|
<a name="979"><span class="lineNum"> 979 </span><span class="lineCov"> 28342 : s = v[nd + 3];</span></a>
|
|
<a name="980"><span class="lineNum"> 980 </span> : </a>
|
|
<a name="981"><span class="lineNum"> 981 </span> : /* special case with only one data disk */</a>
|
|
<a name="982"><span class="lineNum"> 982 </span><span class="lineCov"> 28342 : if (l == 0) {</span></a>
|
|
<a name="983"><span class="lineNum"> 983 </span><span class="lineCov"> 5 : for (i = 0; i < 4; ++i)</span></a>
|
|
<a name="984"><span class="lineNum"> 984 </span><span class="lineCov"> 4 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="985"><span class="lineNum"> 985 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="986"><span class="lineNum"> 986 </span> : }</a>
|
|
<a name="987"><span class="lineNum"> 987 </span> : </a>
|
|
<a name="988"><span class="lineNum"> 988 </span> : raid_sse_begin();</a>
|
|
<a name="989"><span class="lineNum"> 989 </span> : </a>
|
|
<a name="990"><span class="lineNum"> 990 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="991"><span class="lineNum"> 991 </span><span class="lineCov"> 1041309 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="992"><span class="lineNum"> 992 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="993"><span class="lineNum"> 993 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="994"><span class="lineNum"> 994 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="995"><span class="lineNum"> 995 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));</span></a>
|
|
<a name="996"><span class="lineNum"> 996 </span> : </a>
|
|
<a name="997"><span class="lineNum"> 997 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm4,%xmm0");</span></a>
|
|
<a name="998"><span class="lineNum"> 998 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm4,%xmm1");</span></a>
|
|
<a name="999"><span class="lineNum"> 999 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm12,%xmm8");</span></a>
|
|
<a name="1000"><span class="lineNum"> 1000 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm12,%xmm9");</span></a>
|
|
<a name="1001"><span class="lineNum"> 1001 </span> : </a>
|
|
<a name="1002"><span class="lineNum"> 1002 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="1003"><span class="lineNum"> 1003 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm12,%xmm13");</span></a>
|
|
<a name="1004"><span class="lineNum"> 1004 </span><span class="lineCov"> 1012968 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="1005"><span class="lineNum"> 1005 </span><span class="lineCov"> 1012968 : asm volatile ("psrlw $4,%xmm13");</span></a>
|
|
<a name="1006"><span class="lineNum"> 1006 </span><span class="lineCov"> 1012968 : asm volatile ("pand %xmm15,%xmm4");</span></a>
|
|
<a name="1007"><span class="lineNum"> 1007 </span><span class="lineCov"> 1012968 : asm volatile ("pand %xmm15,%xmm12");</span></a>
|
|
<a name="1008"><span class="lineNum"> 1008 </span><span class="lineCov"> 1012968 : asm volatile ("pand %xmm15,%xmm5");</span></a>
|
|
<a name="1009"><span class="lineNum"> 1009 </span><span class="lineCov"> 1012968 : asm volatile ("pand %xmm15,%xmm13");</span></a>
|
|
<a name="1010"><span class="lineNum"> 1010 </span> : </a>
|
|
<a name="1011"><span class="lineNum"> 1011 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1012"><span class="lineNum"> 1012 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1013"><span class="lineNum"> 1013 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm2,%xmm10");</span></a>
|
|
<a name="1014"><span class="lineNum"> 1014 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm7,%xmm15");</span></a>
|
|
<a name="1015"><span class="lineNum"> 1015 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="1016"><span class="lineNum"> 1016 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm12,%xmm10");</span></a>
|
|
<a name="1017"><span class="lineNum"> 1017 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1018"><span class="lineNum"> 1018 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm13,%xmm15");</span></a>
|
|
<a name="1019"><span class="lineNum"> 1019 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="1020"><span class="lineNum"> 1020 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm15,%xmm10");</span></a>
|
|
<a name="1021"><span class="lineNum"> 1021 </span> : </a>
|
|
<a name="1022"><span class="lineNum"> 1022 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1023"><span class="lineNum"> 1023 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1024"><span class="lineNum"> 1024 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm3,%xmm11");</span></a>
|
|
<a name="1025"><span class="lineNum"> 1025 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %xmm7,%xmm15");</span></a>
|
|
<a name="1026"><span class="lineNum"> 1026 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm4,%xmm3");</span></a>
|
|
<a name="1027"><span class="lineNum"> 1027 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm12,%xmm11");</span></a>
|
|
<a name="1028"><span class="lineNum"> 1028 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1029"><span class="lineNum"> 1029 </span><span class="lineCov"> 1012968 : asm volatile ("pshufb %xmm13,%xmm15");</span></a>
|
|
<a name="1030"><span class="lineNum"> 1030 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="1031"><span class="lineNum"> 1031 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm15,%xmm11");</span></a>
|
|
<a name="1032"><span class="lineNum"> 1032 </span> : </a>
|
|
<a name="1033"><span class="lineNum"> 1033 </span> : /* intermediate disks */</a>
|
|
<a name="1034"><span class="lineNum"> 1034 </span><span class="lineCov"> 5278040 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1035"><span class="lineNum"> 1035 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1036"><span class="lineNum"> 1036 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1037"><span class="lineNum"> 1037 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="1038"><span class="lineNum"> 1038 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));</span></a>
|
|
<a name="1039"><span class="lineNum"> 1039 </span> : </a>
|
|
<a name="1040"><span class="lineNum"> 1040 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="1041"><span class="lineNum"> 1041 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm13,%xmm13");</span></a>
|
|
<a name="1042"><span class="lineNum"> 1042 </span><span class="lineCov"> 4265072 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="1043"><span class="lineNum"> 1043 </span><span class="lineCov"> 4265072 : asm volatile ("pcmpgtb %xmm9,%xmm13");</span></a>
|
|
<a name="1044"><span class="lineNum"> 1044 </span><span class="lineCov"> 4265072 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="1045"><span class="lineNum"> 1045 </span><span class="lineCov"> 4265072 : asm volatile ("paddb %xmm9,%xmm9");</span></a>
|
|
<a name="1046"><span class="lineNum"> 1046 </span><span class="lineCov"> 4265072 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1047"><span class="lineNum"> 1047 </span><span class="lineCov"> 4265072 : asm volatile ("pand %xmm7,%xmm13");</span></a>
|
|
<a name="1048"><span class="lineNum"> 1048 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="1049"><span class="lineNum"> 1049 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm13,%xmm9");</span></a>
|
|
<a name="1050"><span class="lineNum"> 1050 </span> : </a>
|
|
<a name="1051"><span class="lineNum"> 1051 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="1052"><span class="lineNum"> 1052 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="1053"><span class="lineNum"> 1053 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm12,%xmm8");</span></a>
|
|
<a name="1054"><span class="lineNum"> 1054 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm12,%xmm9");</span></a>
|
|
<a name="1055"><span class="lineNum"> 1055 </span> : </a>
|
|
<a name="1056"><span class="lineNum"> 1056 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="1057"><span class="lineNum"> 1057 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %xmm12,%xmm13");</span></a>
|
|
<a name="1058"><span class="lineNum"> 1058 </span><span class="lineCov"> 4265072 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="1059"><span class="lineNum"> 1059 </span><span class="lineCov"> 4265072 : asm volatile ("psrlw $4,%xmm13");</span></a>
|
|
<a name="1060"><span class="lineNum"> 1060 </span><span class="lineCov"> 4265072 : asm volatile ("pand %xmm15,%xmm4");</span></a>
|
|
<a name="1061"><span class="lineNum"> 1061 </span><span class="lineCov"> 4265072 : asm volatile ("pand %xmm15,%xmm12");</span></a>
|
|
<a name="1062"><span class="lineNum"> 1062 </span><span class="lineCov"> 4265072 : asm volatile ("pand %xmm15,%xmm5");</span></a>
|
|
<a name="1063"><span class="lineNum"> 1063 </span><span class="lineCov"> 4265072 : asm volatile ("pand %xmm15,%xmm13");</span></a>
|
|
<a name="1064"><span class="lineNum"> 1064 </span> : </a>
|
|
<a name="1065"><span class="lineNum"> 1065 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1066"><span class="lineNum"> 1066 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1067"><span class="lineNum"> 1067 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %xmm6,%xmm14");</span></a>
|
|
<a name="1068"><span class="lineNum"> 1068 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %xmm7,%xmm15");</span></a>
|
|
<a name="1069"><span class="lineNum"> 1069 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="1070"><span class="lineNum"> 1070 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm12,%xmm14");</span></a>
|
|
<a name="1071"><span class="lineNum"> 1071 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1072"><span class="lineNum"> 1072 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm13,%xmm15");</span></a>
|
|
<a name="1073"><span class="lineNum"> 1073 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="1074"><span class="lineNum"> 1074 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm14,%xmm10");</span></a>
|
|
<a name="1075"><span class="lineNum"> 1075 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="1076"><span class="lineNum"> 1076 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm15,%xmm10");</span></a>
|
|
<a name="1077"><span class="lineNum"> 1077 </span> : </a>
|
|
<a name="1078"><span class="lineNum"> 1078 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1079"><span class="lineNum"> 1079 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1080"><span class="lineNum"> 1080 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %xmm6,%xmm14");</span></a>
|
|
<a name="1081"><span class="lineNum"> 1081 </span><span class="lineCov"> 4265072 : asm volatile ("movdqa %xmm7,%xmm15");</span></a>
|
|
<a name="1082"><span class="lineNum"> 1082 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="1083"><span class="lineNum"> 1083 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm12,%xmm14");</span></a>
|
|
<a name="1084"><span class="lineNum"> 1084 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1085"><span class="lineNum"> 1085 </span><span class="lineCov"> 4265072 : asm volatile ("pshufb %xmm13,%xmm15");</span></a>
|
|
<a name="1086"><span class="lineNum"> 1086 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm6,%xmm3");</span></a>
|
|
<a name="1087"><span class="lineNum"> 1087 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm14,%xmm11");</span></a>
|
|
<a name="1088"><span class="lineNum"> 1088 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="1089"><span class="lineNum"> 1089 </span><span class="lineCov"> 4265072 : asm volatile ("pxor %xmm15,%xmm11");</span></a>
|
|
<a name="1090"><span class="lineNum"> 1090 </span> : }</a>
|
|
<a name="1091"><span class="lineNum"> 1091 </span> : </a>
|
|
<a name="1092"><span class="lineNum"> 1092 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1093"><span class="lineNum"> 1093 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1094"><span class="lineNum"> 1094 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1095"><span class="lineNum"> 1095 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="1096"><span class="lineNum"> 1096 </span><span class="lineCov"> 1012968 : asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));</span></a>
|
|
<a name="1097"><span class="lineNum"> 1097 </span> : </a>
|
|
<a name="1098"><span class="lineNum"> 1098 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="1099"><span class="lineNum"> 1099 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm13,%xmm13");</span></a>
|
|
<a name="1100"><span class="lineNum"> 1100 </span><span class="lineCov"> 1012968 : asm volatile ("pcmpgtb %xmm1,%xmm5");</span></a>
|
|
<a name="1101"><span class="lineNum"> 1101 </span><span class="lineCov"> 1012968 : asm volatile ("pcmpgtb %xmm9,%xmm13");</span></a>
|
|
<a name="1102"><span class="lineNum"> 1102 </span><span class="lineCov"> 1012968 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="1103"><span class="lineNum"> 1103 </span><span class="lineCov"> 1012968 : asm volatile ("paddb %xmm9,%xmm9");</span></a>
|
|
<a name="1104"><span class="lineNum"> 1104 </span><span class="lineCov"> 1012968 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1105"><span class="lineNum"> 1105 </span><span class="lineCov"> 1012968 : asm volatile ("pand %xmm7,%xmm13");</span></a>
|
|
<a name="1106"><span class="lineNum"> 1106 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm5,%xmm1");</span></a>
|
|
<a name="1107"><span class="lineNum"> 1107 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm13,%xmm9");</span></a>
|
|
<a name="1108"><span class="lineNum"> 1108 </span> : </a>
|
|
<a name="1109"><span class="lineNum"> 1109 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="1110"><span class="lineNum"> 1110 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="1111"><span class="lineNum"> 1111 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="1112"><span class="lineNum"> 1112 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm4,%xmm3");</span></a>
|
|
<a name="1113"><span class="lineNum"> 1113 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm12,%xmm8");</span></a>
|
|
<a name="1114"><span class="lineNum"> 1114 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm12,%xmm9");</span></a>
|
|
<a name="1115"><span class="lineNum"> 1115 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm12,%xmm10");</span></a>
|
|
<a name="1116"><span class="lineNum"> 1116 </span><span class="lineCov"> 1012968 : asm volatile ("pxor %xmm12,%xmm11");</span></a>
|
|
<a name="1117"><span class="lineNum"> 1117 </span> : </a>
|
|
<a name="1118"><span class="lineNum"> 1118 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1119"><span class="lineNum"> 1119 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16]));</span></a>
|
|
<a name="1120"><span class="lineNum"> 1120 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1121"><span class="lineNum"> 1121 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16]));</span></a>
|
|
<a name="1122"><span class="lineNum"> 1122 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1123"><span class="lineNum"> 1123 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));</span></a>
|
|
<a name="1124"><span class="lineNum"> 1124 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1125"><span class="lineNum"> 1125 </span><span class="lineCov"> 1012968 : asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16]));</span></a>
|
|
<a name="1126"><span class="lineNum"> 1126 </span> : }</a>
|
|
<a name="1127"><span class="lineNum"> 1127 </span> : </a>
|
|
<a name="1128"><span class="lineNum"> 1128 </span> : raid_sse_end();</a>
|
|
<a name="1129"><span class="lineNum"> 1129 </span> : }</a>
|
|
<a name="1130"><span class="lineNum"> 1130 </span> : #endif</a>
|
|
<a name="1131"><span class="lineNum"> 1131 </span> : </a>
|
|
<a name="1132"><span class="lineNum"> 1132 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX2)</a>
|
|
<a name="1133"><span class="lineNum"> 1133 </span> : /*</a>
|
|
<a name="1134"><span class="lineNum"> 1134 </span> : * GEN4 (quad parity with Cauchy matrix) AVX2 implementation</a>
|
|
<a name="1135"><span class="lineNum"> 1135 </span> : *</a>
|
|
<a name="1136"><span class="lineNum"> 1136 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="1137"><span class="lineNum"> 1137 </span> : */</a>
|
|
<a name="1138"><span class="lineNum"> 1138 </span><span class="lineCov"> 6 : void raid_gen4_avx2ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1139"><span class="lineNum"> 1139 </span> : {</a>
|
|
<a name="1140"><span class="lineNum"> 1140 </span><span class="lineCov"> 6 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1141"><span class="lineNum"> 1141 </span> : uint8_t *p;</a>
|
|
<a name="1142"><span class="lineNum"> 1142 </span> : uint8_t *q;</a>
|
|
<a name="1143"><span class="lineNum"> 1143 </span> : uint8_t *r;</a>
|
|
<a name="1144"><span class="lineNum"> 1144 </span> : uint8_t *s;</a>
|
|
<a name="1145"><span class="lineNum"> 1145 </span> : int d, l;</a>
|
|
<a name="1146"><span class="lineNum"> 1146 </span> : size_t i;</a>
|
|
<a name="1147"><span class="lineNum"> 1147 </span> : </a>
|
|
<a name="1148"><span class="lineNum"> 1148 </span><span class="lineCov"> 6 : l = nd - 1;</span></a>
|
|
<a name="1149"><span class="lineNum"> 1149 </span><span class="lineCov"> 6 : p = v[nd];</span></a>
|
|
<a name="1150"><span class="lineNum"> 1150 </span><span class="lineCov"> 6 : q = v[nd + 1];</span></a>
|
|
<a name="1151"><span class="lineNum"> 1151 </span><span class="lineCov"> 6 : r = v[nd + 2];</span></a>
|
|
<a name="1152"><span class="lineNum"> 1152 </span><span class="lineCov"> 6 : s = v[nd + 3];</span></a>
|
|
<a name="1153"><span class="lineNum"> 1153 </span> : </a>
|
|
<a name="1154"><span class="lineNum"> 1154 </span> : /* special case with only one data disk */</a>
|
|
<a name="1155"><span class="lineNum"> 1155 </span><span class="lineCov"> 6 : if (l == 0) {</span></a>
|
|
<a name="1156"><span class="lineNum"> 1156 </span><span class="lineCov"> 5 : for (i = 0; i < 4; ++i)</span></a>
|
|
<a name="1157"><span class="lineNum"> 1157 </span><span class="lineCov"> 4 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1158"><span class="lineNum"> 1158 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1159"><span class="lineNum"> 1159 </span> : }</a>
|
|
<a name="1160"><span class="lineNum"> 1160 </span> : </a>
|
|
<a name="1161"><span class="lineNum"> 1161 </span> : raid_avx_begin();</a>
|
|
<a name="1162"><span class="lineNum"> 1162 </span> : </a>
|
|
<a name="1163"><span class="lineNum"> 1163 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1164"><span class="lineNum"> 1164 </span><span class="lineCov"> 16393 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="1165"><span class="lineNum"> 1165 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1166"><span class="lineNum"> 1166 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1167"><span class="lineNum"> 1167 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="1168"><span class="lineNum"> 1168 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));</span></a>
|
|
<a name="1169"><span class="lineNum"> 1169 </span> : </a>
|
|
<a name="1170"><span class="lineNum"> 1170 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %ymm4,%ymm0");</span></a>
|
|
<a name="1171"><span class="lineNum"> 1171 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %ymm4,%ymm1");</span></a>
|
|
<a name="1172"><span class="lineNum"> 1172 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %ymm12,%ymm8");</span></a>
|
|
<a name="1173"><span class="lineNum"> 1173 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %ymm12,%ymm9");</span></a>
|
|
<a name="1174"><span class="lineNum"> 1174 </span> : </a>
|
|
<a name="1175"><span class="lineNum"> 1175 </span><span class="lineCov"> 16388 : asm volatile ("vpsrlw $4,%ymm4,%ymm5");</span></a>
|
|
<a name="1176"><span class="lineNum"> 1176 </span><span class="lineCov"> 16388 : asm volatile ("vpsrlw $4,%ymm12,%ymm13");</span></a>
|
|
<a name="1177"><span class="lineNum"> 1177 </span><span class="lineCov"> 16388 : asm volatile ("vpand %ymm15,%ymm4,%ymm4");</span></a>
|
|
<a name="1178"><span class="lineNum"> 1178 </span><span class="lineCov"> 16388 : asm volatile ("vpand %ymm15,%ymm12,%ymm12");</span></a>
|
|
<a name="1179"><span class="lineNum"> 1179 </span><span class="lineCov"> 16388 : asm volatile ("vpand %ymm15,%ymm5,%ymm5");</span></a>
|
|
<a name="1180"><span class="lineNum"> 1180 </span><span class="lineCov"> 16388 : asm volatile ("vpand %ymm15,%ymm13,%ymm13");</span></a>
|
|
<a name="1181"><span class="lineNum"> 1181 </span> : </a>
|
|
<a name="1182"><span class="lineNum"> 1182 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1183"><span class="lineNum"> 1183 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1184"><span class="lineNum"> 1184 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");</span></a>
|
|
<a name="1185"><span class="lineNum"> 1185 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");</span></a>
|
|
<a name="1186"><span class="lineNum"> 1186 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm12,%ymm10,%ymm10");</span></a>
|
|
<a name="1187"><span class="lineNum"> 1187 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");</span></a>
|
|
<a name="1188"><span class="lineNum"> 1188 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm7,%ymm2,%ymm2");</span></a>
|
|
<a name="1189"><span class="lineNum"> 1189 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="1190"><span class="lineNum"> 1190 </span> : </a>
|
|
<a name="1191"><span class="lineNum"> 1191 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1192"><span class="lineNum"> 1192 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1193"><span class="lineNum"> 1193 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm4,%ymm11,%ymm3");</span></a>
|
|
<a name="1194"><span class="lineNum"> 1194 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");</span></a>
|
|
<a name="1195"><span class="lineNum"> 1195 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm12,%ymm11,%ymm11");</span></a>
|
|
<a name="1196"><span class="lineNum"> 1196 </span><span class="lineCov"> 16388 : asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");</span></a>
|
|
<a name="1197"><span class="lineNum"> 1197 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm7,%ymm3,%ymm3");</span></a>
|
|
<a name="1198"><span class="lineNum"> 1198 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm15,%ymm11,%ymm11");</span></a>
|
|
<a name="1199"><span class="lineNum"> 1199 </span> : </a>
|
|
<a name="1200"><span class="lineNum"> 1200 </span> : /* intermediate disks */</a>
|
|
<a name="1201"><span class="lineNum"> 1201 </span><span class="lineCov"> 114812 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1202"><span class="lineNum"> 1202 </span><span class="lineCov"> 98424 : asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1203"><span class="lineNum"> 1203 </span><span class="lineCov"> 98424 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1204"><span class="lineNum"> 1204 </span><span class="lineCov"> 98424 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="1205"><span class="lineNum"> 1205 </span><span class="lineCov"> 98424 : asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));</span></a>
|
|
<a name="1206"><span class="lineNum"> 1206 </span> : </a>
|
|
<a name="1207"><span class="lineNum"> 1207 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm5,%ymm5,%ymm5");</span></a>
|
|
<a name="1208"><span class="lineNum"> 1208 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm13,%ymm13,%ymm13");</span></a>
|
|
<a name="1209"><span class="lineNum"> 1209 </span><span class="lineCov"> 98424 : asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");</span></a>
|
|
<a name="1210"><span class="lineNum"> 1210 </span><span class="lineCov"> 98424 : asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");</span></a>
|
|
<a name="1211"><span class="lineNum"> 1211 </span><span class="lineCov"> 98424 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="1212"><span class="lineNum"> 1212 </span><span class="lineCov"> 98424 : asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");</span></a>
|
|
<a name="1213"><span class="lineNum"> 1213 </span><span class="lineCov"> 98424 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="1214"><span class="lineNum"> 1214 </span><span class="lineCov"> 98424 : asm volatile ("vpand %ymm7,%ymm13,%ymm13");</span></a>
|
|
<a name="1215"><span class="lineNum"> 1215 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm5,%ymm1,%ymm1");</span></a>
|
|
<a name="1216"><span class="lineNum"> 1216 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm13,%ymm9,%ymm9");</span></a>
|
|
<a name="1217"><span class="lineNum"> 1217 </span> : </a>
|
|
<a name="1218"><span class="lineNum"> 1218 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm4,%ymm0,%ymm0");</span></a>
|
|
<a name="1219"><span class="lineNum"> 1219 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm4,%ymm1,%ymm1");</span></a>
|
|
<a name="1220"><span class="lineNum"> 1220 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm12,%ymm8,%ymm8");</span></a>
|
|
<a name="1221"><span class="lineNum"> 1221 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm12,%ymm9,%ymm9");</span></a>
|
|
<a name="1222"><span class="lineNum"> 1222 </span> : </a>
|
|
<a name="1223"><span class="lineNum"> 1223 </span><span class="lineCov"> 98424 : asm volatile ("vpsrlw $4,%ymm4,%ymm5");</span></a>
|
|
<a name="1224"><span class="lineNum"> 1224 </span><span class="lineCov"> 98424 : asm volatile ("vpsrlw $4,%ymm12,%ymm13");</span></a>
|
|
<a name="1225"><span class="lineNum"> 1225 </span><span class="lineCov"> 98424 : asm volatile ("vpand %ymm15,%ymm4,%ymm4");</span></a>
|
|
<a name="1226"><span class="lineNum"> 1226 </span><span class="lineCov"> 98424 : asm volatile ("vpand %ymm15,%ymm12,%ymm12");</span></a>
|
|
<a name="1227"><span class="lineNum"> 1227 </span><span class="lineCov"> 98424 : asm volatile ("vpand %ymm15,%ymm5,%ymm5");</span></a>
|
|
<a name="1228"><span class="lineNum"> 1228 </span><span class="lineCov"> 98424 : asm volatile ("vpand %ymm15,%ymm13,%ymm13");</span></a>
|
|
<a name="1229"><span class="lineNum"> 1229 </span> : </a>
|
|
<a name="1230"><span class="lineNum"> 1230 </span><span class="lineCov"> 98424 : asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1231"><span class="lineNum"> 1231 </span><span class="lineCov"> 98424 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1232"><span class="lineNum"> 1232 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");</span></a>
|
|
<a name="1233"><span class="lineNum"> 1233 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");</span></a>
|
|
<a name="1234"><span class="lineNum"> 1234 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");</span></a>
|
|
<a name="1235"><span class="lineNum"> 1235 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");</span></a>
|
|
<a name="1236"><span class="lineNum"> 1236 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm6,%ymm2,%ymm2");</span></a>
|
|
<a name="1237"><span class="lineNum"> 1237 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm14,%ymm10,%ymm10");</span></a>
|
|
<a name="1238"><span class="lineNum"> 1238 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm7,%ymm2,%ymm2");</span></a>
|
|
<a name="1239"><span class="lineNum"> 1239 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="1240"><span class="lineNum"> 1240 </span> : </a>
|
|
<a name="1241"><span class="lineNum"> 1241 </span><span class="lineCov"> 98424 : asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1242"><span class="lineNum"> 1242 </span><span class="lineCov"> 98424 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1243"><span class="lineNum"> 1243 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm4,%ymm14,%ymm6");</span></a>
|
|
<a name="1244"><span class="lineNum"> 1244 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm5,%ymm15,%ymm7");</span></a>
|
|
<a name="1245"><span class="lineNum"> 1245 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");</span></a>
|
|
<a name="1246"><span class="lineNum"> 1246 </span><span class="lineCov"> 98424 : asm volatile ("vpshufb %ymm13,%ymm15,%ymm15");</span></a>
|
|
<a name="1247"><span class="lineNum"> 1247 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm6,%ymm3,%ymm3");</span></a>
|
|
<a name="1248"><span class="lineNum"> 1248 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm14,%ymm11,%ymm11");</span></a>
|
|
<a name="1249"><span class="lineNum"> 1249 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm7,%ymm3,%ymm3");</span></a>
|
|
<a name="1250"><span class="lineNum"> 1250 </span><span class="lineCov"> 98424 : asm volatile ("vpxor %ymm15,%ymm11,%ymm11");</span></a>
|
|
<a name="1251"><span class="lineNum"> 1251 </span> : }</a>
|
|
<a name="1252"><span class="lineNum"> 1252 </span> : </a>
|
|
<a name="1253"><span class="lineNum"> 1253 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1254"><span class="lineNum"> 1254 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1255"><span class="lineNum"> 1255 </span><span class="lineCov"> 16388 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1256"><span class="lineNum"> 1256 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="1257"><span class="lineNum"> 1257 </span><span class="lineCov"> 16388 : asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));</span></a>
|
|
<a name="1258"><span class="lineNum"> 1258 </span> : </a>
|
|
<a name="1259"><span class="lineNum"> 1259 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm5,%ymm5,%ymm5");</span></a>
|
|
<a name="1260"><span class="lineNum"> 1260 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm13,%ymm13,%ymm13");</span></a>
|
|
<a name="1261"><span class="lineNum"> 1261 </span><span class="lineCov"> 16388 : asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");</span></a>
|
|
<a name="1262"><span class="lineNum"> 1262 </span><span class="lineCov"> 16388 : asm volatile ("vpcmpgtb %ymm9,%ymm13,%ymm13");</span></a>
|
|
<a name="1263"><span class="lineNum"> 1263 </span><span class="lineCov"> 16388 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="1264"><span class="lineNum"> 1264 </span><span class="lineCov"> 16388 : asm volatile ("vpaddb %ymm9,%ymm9,%ymm9");</span></a>
|
|
<a name="1265"><span class="lineNum"> 1265 </span><span class="lineCov"> 16388 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="1266"><span class="lineNum"> 1266 </span><span class="lineCov"> 16388 : asm volatile ("vpand %ymm7,%ymm13,%ymm13");</span></a>
|
|
<a name="1267"><span class="lineNum"> 1267 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm5,%ymm1,%ymm1");</span></a>
|
|
<a name="1268"><span class="lineNum"> 1268 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm13,%ymm9,%ymm9");</span></a>
|
|
<a name="1269"><span class="lineNum"> 1269 </span> : </a>
|
|
<a name="1270"><span class="lineNum"> 1270 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm4,%ymm0,%ymm0");</span></a>
|
|
<a name="1271"><span class="lineNum"> 1271 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm4,%ymm1,%ymm1");</span></a>
|
|
<a name="1272"><span class="lineNum"> 1272 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="1273"><span class="lineNum"> 1273 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm4,%ymm3,%ymm3");</span></a>
|
|
<a name="1274"><span class="lineNum"> 1274 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm12,%ymm8,%ymm8");</span></a>
|
|
<a name="1275"><span class="lineNum"> 1275 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm12,%ymm9,%ymm9");</span></a>
|
|
<a name="1276"><span class="lineNum"> 1276 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm12,%ymm10,%ymm10");</span></a>
|
|
<a name="1277"><span class="lineNum"> 1277 </span><span class="lineCov"> 16388 : asm volatile ("vpxor %ymm12,%ymm11,%ymm11");</span></a>
|
|
<a name="1278"><span class="lineNum"> 1278 </span> : </a>
|
|
<a name="1279"><span class="lineNum"> 1279 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1280"><span class="lineNum"> 1280 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm8,%0" : "=m" (p[i + 32]));</span></a>
|
|
<a name="1281"><span class="lineNum"> 1281 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1282"><span class="lineNum"> 1282 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm9,%0" : "=m" (q[i + 32]));</span></a>
|
|
<a name="1283"><span class="lineNum"> 1283 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1284"><span class="lineNum"> 1284 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32]));</span></a>
|
|
<a name="1285"><span class="lineNum"> 1285 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1286"><span class="lineNum"> 1286 </span><span class="lineCov"> 16388 : asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32]));</span></a>
|
|
<a name="1287"><span class="lineNum"> 1287 </span> : }</a>
|
|
<a name="1288"><span class="lineNum"> 1288 </span> : </a>
|
|
<a name="1289"><span class="lineNum"> 1289 </span> : raid_avx_end();</a>
|
|
<a name="1290"><span class="lineNum"> 1290 </span> : }</a>
|
|
<a name="1291"><span class="lineNum"> 1291 </span> : #endif</a>
|
|
<a name="1292"><span class="lineNum"> 1292 </span> : </a>
|
|
<a name="1293"><span class="lineNum"> 1293 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="1294"><span class="lineNum"> 1294 </span> : /*</a>
|
|
<a name="1295"><span class="lineNum"> 1295 </span> : * GEN4 (quad parity with Cauchy matrix) AVX512BW implementation</a>
|
|
<a name="1296"><span class="lineNum"> 1296 </span> : */</a>
|
|
<a name="1297"><span class="lineNum"> 1297 </span><span class="lineCov"> 5470 : void raid_gen4_avx512bw(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1298"><span class="lineNum"> 1298 </span> : {</a>
|
|
<a name="1299"><span class="lineNum"> 1299 </span><span class="lineCov"> 5470 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1300"><span class="lineNum"> 1300 </span> : uint8_t *p;</a>
|
|
<a name="1301"><span class="lineNum"> 1301 </span> : uint8_t *q;</a>
|
|
<a name="1302"><span class="lineNum"> 1302 </span> : uint8_t *r;</a>
|
|
<a name="1303"><span class="lineNum"> 1303 </span> : uint8_t *s;</a>
|
|
<a name="1304"><span class="lineNum"> 1304 </span> : int d, l;</a>
|
|
<a name="1305"><span class="lineNum"> 1305 </span> : size_t i;</a>
|
|
<a name="1306"><span class="lineNum"> 1306 </span> : </a>
|
|
<a name="1307"><span class="lineNum"> 1307 </span><span class="lineCov"> 5470 : l = nd - 1;</span></a>
|
|
<a name="1308"><span class="lineNum"> 1308 </span><span class="lineCov"> 5470 : p = v[nd];</span></a>
|
|
<a name="1309"><span class="lineNum"> 1309 </span><span class="lineCov"> 5470 : q = v[nd + 1];</span></a>
|
|
<a name="1310"><span class="lineNum"> 1310 </span><span class="lineCov"> 5470 : r = v[nd + 2];</span></a>
|
|
<a name="1311"><span class="lineNum"> 1311 </span><span class="lineCov"> 5470 : s = v[nd + 3];</span></a>
|
|
<a name="1312"><span class="lineNum"> 1312 </span> : </a>
|
|
<a name="1313"><span class="lineNum"> 1313 </span> : /* special case with only one data disk */</a>
|
|
<a name="1314"><span class="lineNum"> 1314 </span><span class="lineCov"> 5470 : if (l == 0) {</span></a>
|
|
<a name="1315"><span class="lineNum"> 1315 </span><span class="lineCov"> 5 : for (i = 0; i < 4; ++i)</span></a>
|
|
<a name="1316"><span class="lineNum"> 1316 </span><span class="lineCov"> 4 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1317"><span class="lineNum"> 1317 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1318"><span class="lineNum"> 1318 </span> : }</a>
|
|
<a name="1319"><span class="lineNum"> 1319 </span> : </a>
|
|
<a name="1320"><span class="lineNum"> 1320 </span> : raid_avx_begin();</a>
|
|
<a name="1321"><span class="lineNum"> 1321 </span> : </a>
|
|
<a name="1322"><span class="lineNum"> 1322 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1323"><span class="lineNum"> 1323 </span><span class="lineCov"> 5469 : asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="1324"><span class="lineNum"> 1324 </span><span class="lineCov"> 5469 : asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1325"><span class="lineNum"> 1325 </span><span class="lineCov"> 5469 : asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1326"><span class="lineNum"> 1326 </span> : </a>
|
|
<a name="1327"><span class="lineNum"> 1327 </span><span class="lineCov"> 35889 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="1328"><span class="lineNum"> 1328 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1329"><span class="lineNum"> 1329 </span><span class="lineCov"> 30420 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="1330"><span class="lineNum"> 1330 </span> : </a>
|
|
<a name="1331"><span class="lineNum"> 1331 </span><span class="lineCov"> 30420 : asm volatile ("vmovdqa64 %zmm10,%zmm0");</span></a>
|
|
<a name="1332"><span class="lineNum"> 1332 </span><span class="lineCov"> 30420 : asm volatile ("vmovdqa64 %zmm10,%zmm1");</span></a>
|
|
<a name="1333"><span class="lineNum"> 1333 </span> : </a>
|
|
<a name="1334"><span class="lineNum"> 1334 </span><span class="lineCov"> 30420 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="1335"><span class="lineNum"> 1335 </span><span class="lineCov"> 30420 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="1336"><span class="lineNum"> 1336 </span><span class="lineCov"> 30420 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="1337"><span class="lineNum"> 1337 </span> : </a>
|
|
<a name="1338"><span class="lineNum"> 1338 </span><span class="lineCov"> 30420 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1339"><span class="lineNum"> 1339 </span><span class="lineCov"> 30420 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1340"><span class="lineNum"> 1340 </span><span class="lineCov"> 30420 : asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="1341"><span class="lineNum"> 1341 </span><span class="lineCov"> 30420 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1342"><span class="lineNum"> 1342 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="1343"><span class="lineNum"> 1343 </span> : </a>
|
|
<a name="1344"><span class="lineNum"> 1344 </span><span class="lineCov"> 30420 : asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1345"><span class="lineNum"> 1345 </span><span class="lineCov"> 30420 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1346"><span class="lineNum"> 1346 </span><span class="lineCov"> 30420 : asm volatile ("vpshufb %zmm10,%zmm3,%zmm3");</span></a>
|
|
<a name="1347"><span class="lineNum"> 1347 </span><span class="lineCov"> 30420 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1348"><span class="lineNum"> 1348 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");</span></a>
|
|
<a name="1349"><span class="lineNum"> 1349 </span> : </a>
|
|
<a name="1350"><span class="lineNum"> 1350 </span> : /* intermediate disks */</a>
|
|
<a name="1351"><span class="lineNum"> 1351 </span><span class="lineCov"> 303468 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1352"><span class="lineNum"> 1352 </span><span class="lineCov"> 273048 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="1353"><span class="lineNum"> 1353 </span> : </a>
|
|
<a name="1354"><span class="lineNum"> 1354 </span><span class="lineCov"> 273048 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="1355"><span class="lineNum"> 1355 </span><span class="lineCov"> 273048 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="1356"><span class="lineNum"> 1356 </span><span class="lineCov"> 273048 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="1357"><span class="lineNum"> 1357 </span><span class="lineCov"> 273048 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="1358"><span class="lineNum"> 1358 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="1359"><span class="lineNum"> 1359 </span> : </a>
|
|
<a name="1360"><span class="lineNum"> 1360 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="1361"><span class="lineNum"> 1361 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="1362"><span class="lineNum"> 1362 </span> : </a>
|
|
<a name="1363"><span class="lineNum"> 1363 </span><span class="lineCov"> 273048 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="1364"><span class="lineNum"> 1364 </span><span class="lineCov"> 273048 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="1365"><span class="lineNum"> 1365 </span><span class="lineCov"> 273048 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="1366"><span class="lineNum"> 1366 </span> : </a>
|
|
<a name="1367"><span class="lineNum"> 1367 </span><span class="lineCov"> 273048 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1368"><span class="lineNum"> 1368 </span><span class="lineCov"> 273048 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1369"><span class="lineNum"> 1369 </span><span class="lineCov"> 273048 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="1370"><span class="lineNum"> 1370 </span><span class="lineCov"> 273048 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1371"><span class="lineNum"> 1371 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");</span></a>
|
|
<a name="1372"><span class="lineNum"> 1372 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="1373"><span class="lineNum"> 1373 </span> : </a>
|
|
<a name="1374"><span class="lineNum"> 1374 </span><span class="lineCov"> 273048 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1375"><span class="lineNum"> 1375 </span><span class="lineCov"> 273048 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1376"><span class="lineNum"> 1376 </span><span class="lineCov"> 273048 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="1377"><span class="lineNum"> 1377 </span><span class="lineCov"> 273048 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1378"><span class="lineNum"> 1378 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm12,%zmm3,%zmm3");</span></a>
|
|
<a name="1379"><span class="lineNum"> 1379 </span><span class="lineCov"> 273048 : asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");</span></a>
|
|
<a name="1380"><span class="lineNum"> 1380 </span> : }</a>
|
|
<a name="1381"><span class="lineNum"> 1381 </span> : </a>
|
|
<a name="1382"><span class="lineNum"> 1382 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1383"><span class="lineNum"> 1383 </span><span class="lineCov"> 30420 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="1384"><span class="lineNum"> 1384 </span> : </a>
|
|
<a name="1385"><span class="lineNum"> 1385 </span><span class="lineCov"> 30420 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="1386"><span class="lineNum"> 1386 </span><span class="lineCov"> 30420 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="1387"><span class="lineNum"> 1387 </span><span class="lineCov"> 30420 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="1388"><span class="lineNum"> 1388 </span><span class="lineCov"> 30420 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="1389"><span class="lineNum"> 1389 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="1390"><span class="lineNum"> 1390 </span> : </a>
|
|
<a name="1391"><span class="lineNum"> 1391 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="1392"><span class="lineNum"> 1392 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="1393"><span class="lineNum"> 1393 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="1394"><span class="lineNum"> 1394 </span><span class="lineCov"> 30420 : asm volatile ("vpxorq %zmm10,%zmm3,%zmm3");</span></a>
|
|
<a name="1395"><span class="lineNum"> 1395 </span> : </a>
|
|
<a name="1396"><span class="lineNum"> 1396 </span><span class="lineCov"> 30420 : asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1397"><span class="lineNum"> 1397 </span><span class="lineCov"> 30420 : asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1398"><span class="lineNum"> 1398 </span><span class="lineCov"> 30420 : asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1399"><span class="lineNum"> 1399 </span><span class="lineCov"> 30420 : asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1400"><span class="lineNum"> 1400 </span> : }</a>
|
|
<a name="1401"><span class="lineNum"> 1401 </span> : </a>
|
|
<a name="1402"><span class="lineNum"> 1402 </span> : raid_avx_end();</a>
|
|
<a name="1403"><span class="lineNum"> 1403 </span> : }</a>
|
|
<a name="1404"><span class="lineNum"> 1404 </span> : #endif</a>
|
|
<a name="1405"><span class="lineNum"> 1405 </span> : </a>
|
|
<a name="1406"><span class="lineNum"> 1406 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="1407"><span class="lineNum"> 1407 </span> : /*</a>
|
|
<a name="1408"><span class="lineNum"> 1408 </span> : * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="1409"><span class="lineNum"> 1409 </span> : */</a>
|
|
<a name="1410"><span class="lineNum"> 1410 </span><span class="lineCov"> 10 : void raid_gen5_ssse3(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1411"><span class="lineNum"> 1411 </span> : {</a>
|
|
<a name="1412"><span class="lineNum"> 1412 </span><span class="lineCov"> 10 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1413"><span class="lineNum"> 1413 </span> : uint8_t *p;</a>
|
|
<a name="1414"><span class="lineNum"> 1414 </span> : uint8_t *q;</a>
|
|
<a name="1415"><span class="lineNum"> 1415 </span> : uint8_t *r;</a>
|
|
<a name="1416"><span class="lineNum"> 1416 </span> : uint8_t *s;</a>
|
|
<a name="1417"><span class="lineNum"> 1417 </span> : uint8_t *t;</a>
|
|
<a name="1418"><span class="lineNum"> 1418 </span> : int d, l;</a>
|
|
<a name="1419"><span class="lineNum"> 1419 </span> : size_t i;</a>
|
|
<a name="1420"><span class="lineNum"> 1420 </span> : uint8_t buffer[16+16];</a>
|
|
<a name="1421"><span class="lineNum"> 1421 </span><span class="lineCov"> 10 : uint8_t *pd = __align_ptr(buffer, 16);</span></a>
|
|
<a name="1422"><span class="lineNum"> 1422 </span> : </a>
|
|
<a name="1423"><span class="lineNum"> 1423 </span><span class="lineCov"> 10 : l = nd - 1;</span></a>
|
|
<a name="1424"><span class="lineNum"> 1424 </span><span class="lineCov"> 10 : p = v[nd];</span></a>
|
|
<a name="1425"><span class="lineNum"> 1425 </span><span class="lineCov"> 10 : q = v[nd + 1];</span></a>
|
|
<a name="1426"><span class="lineNum"> 1426 </span><span class="lineCov"> 10 : r = v[nd + 2];</span></a>
|
|
<a name="1427"><span class="lineNum"> 1427 </span><span class="lineCov"> 10 : s = v[nd + 3];</span></a>
|
|
<a name="1428"><span class="lineNum"> 1428 </span><span class="lineCov"> 10 : t = v[nd + 4];</span></a>
|
|
<a name="1429"><span class="lineNum"> 1429 </span> : </a>
|
|
<a name="1430"><span class="lineNum"> 1430 </span> : /* special case with only one data disk */</a>
|
|
<a name="1431"><span class="lineNum"> 1431 </span><span class="lineCov"> 10 : if (l == 0) {</span></a>
|
|
<a name="1432"><span class="lineNum"> 1432 </span><span class="lineCov"> 6 : for (i = 0; i < 5; ++i)</span></a>
|
|
<a name="1433"><span class="lineNum"> 1433 </span><span class="lineCov"> 5 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1434"><span class="lineNum"> 1434 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1435"><span class="lineNum"> 1435 </span> : }</a>
|
|
<a name="1436"><span class="lineNum"> 1436 </span> : </a>
|
|
<a name="1437"><span class="lineNum"> 1437 </span> : raid_sse_begin();</a>
|
|
<a name="1438"><span class="lineNum"> 1438 </span> : </a>
|
|
<a name="1439"><span class="lineNum"> 1439 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1440"><span class="lineNum"> 1440 </span><span class="lineCov"> 131097 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="1441"><span class="lineNum"> 1441 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1442"><span class="lineNum"> 1442 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="1443"><span class="lineNum"> 1443 </span> : </a>
|
|
<a name="1444"><span class="lineNum"> 1444 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %xmm4,%xmm0");</span></a>
|
|
<a name="1445"><span class="lineNum"> 1445 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));</span></a>
|
|
<a name="1446"><span class="lineNum"> 1446 </span> : </a>
|
|
<a name="1447"><span class="lineNum"> 1447 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1448"><span class="lineNum"> 1448 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="1449"><span class="lineNum"> 1449 </span><span class="lineCov"> 131088 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="1450"><span class="lineNum"> 1450 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="1451"><span class="lineNum"> 1451 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1452"><span class="lineNum"> 1452 </span> : </a>
|
|
<a name="1453"><span class="lineNum"> 1453 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1454"><span class="lineNum"> 1454 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1455"><span class="lineNum"> 1455 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm1");</span></a>
|
|
<a name="1456"><span class="lineNum"> 1456 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1457"><span class="lineNum"> 1457 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm1");</span></a>
|
|
<a name="1458"><span class="lineNum"> 1458 </span> : </a>
|
|
<a name="1459"><span class="lineNum"> 1459 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1460"><span class="lineNum"> 1460 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1461"><span class="lineNum"> 1461 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="1462"><span class="lineNum"> 1462 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1463"><span class="lineNum"> 1463 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="1464"><span class="lineNum"> 1464 </span> : </a>
|
|
<a name="1465"><span class="lineNum"> 1465 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="1466"><span class="lineNum"> 1466 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="1467"><span class="lineNum"> 1467 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm3");</span></a>
|
|
<a name="1468"><span class="lineNum"> 1468 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1469"><span class="lineNum"> 1469 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="1470"><span class="lineNum"> 1470 </span> : </a>
|
|
<a name="1471"><span class="lineNum"> 1471 </span> : /* intermediate disks */</a>
|
|
<a name="1472"><span class="lineNum"> 1472 </span><span class="lineCov"> 918000 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1473"><span class="lineNum"> 1473 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="1474"><span class="lineNum"> 1474 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));</span></a>
|
|
<a name="1475"><span class="lineNum"> 1475 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1476"><span class="lineNum"> 1476 </span> : </a>
|
|
<a name="1477"><span class="lineNum"> 1477 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="1478"><span class="lineNum"> 1478 </span><span class="lineCov"> 786912 : asm volatile ("pcmpgtb %xmm0,%xmm5");</span></a>
|
|
<a name="1479"><span class="lineNum"> 1479 </span><span class="lineCov"> 786912 : asm volatile ("paddb %xmm0,%xmm0");</span></a>
|
|
<a name="1480"><span class="lineNum"> 1480 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1481"><span class="lineNum"> 1481 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm5,%xmm0");</span></a>
|
|
<a name="1482"><span class="lineNum"> 1482 </span> : </a>
|
|
<a name="1483"><span class="lineNum"> 1483 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="1484"><span class="lineNum"> 1484 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm6");</span></a>
|
|
<a name="1485"><span class="lineNum"> 1485 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0]));</span></a>
|
|
<a name="1486"><span class="lineNum"> 1486 </span> : </a>
|
|
<a name="1487"><span class="lineNum"> 1487 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1488"><span class="lineNum"> 1488 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="1489"><span class="lineNum"> 1489 </span><span class="lineCov"> 786912 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="1490"><span class="lineNum"> 1490 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="1491"><span class="lineNum"> 1491 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1492"><span class="lineNum"> 1492 </span> : </a>
|
|
<a name="1493"><span class="lineNum"> 1493 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1494"><span class="lineNum"> 1494 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1495"><span class="lineNum"> 1495 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="1496"><span class="lineNum"> 1496 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1497"><span class="lineNum"> 1497 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm1");</span></a>
|
|
<a name="1498"><span class="lineNum"> 1498 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm1");</span></a>
|
|
<a name="1499"><span class="lineNum"> 1499 </span> : </a>
|
|
<a name="1500"><span class="lineNum"> 1500 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1501"><span class="lineNum"> 1501 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1502"><span class="lineNum"> 1502 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="1503"><span class="lineNum"> 1503 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1504"><span class="lineNum"> 1504 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="1505"><span class="lineNum"> 1505 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="1506"><span class="lineNum"> 1506 </span> : </a>
|
|
<a name="1507"><span class="lineNum"> 1507 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="1508"><span class="lineNum"> 1508 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="1509"><span class="lineNum"> 1509 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="1510"><span class="lineNum"> 1510 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1511"><span class="lineNum"> 1511 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm3");</span></a>
|
|
<a name="1512"><span class="lineNum"> 1512 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="1513"><span class="lineNum"> 1513 </span> : }</a>
|
|
<a name="1514"><span class="lineNum"> 1514 </span> : </a>
|
|
<a name="1515"><span class="lineNum"> 1515 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1516"><span class="lineNum"> 1516 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="1517"><span class="lineNum"> 1517 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));</span></a>
|
|
<a name="1518"><span class="lineNum"> 1518 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1519"><span class="lineNum"> 1519 </span> : </a>
|
|
<a name="1520"><span class="lineNum"> 1520 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm5,%xmm5");</span></a>
|
|
<a name="1521"><span class="lineNum"> 1521 </span><span class="lineCov"> 131088 : asm volatile ("pcmpgtb %xmm0,%xmm5");</span></a>
|
|
<a name="1522"><span class="lineNum"> 1522 </span><span class="lineCov"> 131088 : asm volatile ("paddb %xmm0,%xmm0");</span></a>
|
|
<a name="1523"><span class="lineNum"> 1523 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1524"><span class="lineNum"> 1524 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm5,%xmm0");</span></a>
|
|
<a name="1525"><span class="lineNum"> 1525 </span> : </a>
|
|
<a name="1526"><span class="lineNum"> 1526 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="1527"><span class="lineNum"> 1527 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="1528"><span class="lineNum"> 1528 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="1529"><span class="lineNum"> 1529 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm3");</span></a>
|
|
<a name="1530"><span class="lineNum"> 1530 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm6");</span></a>
|
|
<a name="1531"><span class="lineNum"> 1531 </span> : </a>
|
|
<a name="1532"><span class="lineNum"> 1532 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1533"><span class="lineNum"> 1533 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1534"><span class="lineNum"> 1534 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1535"><span class="lineNum"> 1535 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1536"><span class="lineNum"> 1536 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i]));</span></a>
|
|
<a name="1537"><span class="lineNum"> 1537 </span> : }</a>
|
|
<a name="1538"><span class="lineNum"> 1538 </span> : </a>
|
|
<a name="1539"><span class="lineNum"> 1539 </span> : raid_sse_end();</a>
|
|
<a name="1540"><span class="lineNum"> 1540 </span> : }</a>
|
|
<a name="1541"><span class="lineNum"> 1541 </span> : #endif</a>
|
|
<a name="1542"><span class="lineNum"> 1542 </span> : </a>
|
|
<a name="1543"><span class="lineNum"> 1543 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_SSSE3)</a>
|
|
<a name="1544"><span class="lineNum"> 1544 </span> : /*</a>
|
|
<a name="1545"><span class="lineNum"> 1545 </span> : * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="1546"><span class="lineNum"> 1546 </span> : *</a>
|
|
<a name="1547"><span class="lineNum"> 1547 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="1548"><span class="lineNum"> 1548 </span> : */</a>
|
|
<a name="1549"><span class="lineNum"> 1549 </span><span class="lineCov"> 37382 : void raid_gen5_ssse3ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1550"><span class="lineNum"> 1550 </span> : {</a>
|
|
<a name="1551"><span class="lineNum"> 1551 </span><span class="lineCov"> 37382 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1552"><span class="lineNum"> 1552 </span> : uint8_t *p;</a>
|
|
<a name="1553"><span class="lineNum"> 1553 </span> : uint8_t *q;</a>
|
|
<a name="1554"><span class="lineNum"> 1554 </span> : uint8_t *r;</a>
|
|
<a name="1555"><span class="lineNum"> 1555 </span> : uint8_t *s;</a>
|
|
<a name="1556"><span class="lineNum"> 1556 </span> : uint8_t *t;</a>
|
|
<a name="1557"><span class="lineNum"> 1557 </span> : int d, l;</a>
|
|
<a name="1558"><span class="lineNum"> 1558 </span> : size_t i;</a>
|
|
<a name="1559"><span class="lineNum"> 1559 </span> : </a>
|
|
<a name="1560"><span class="lineNum"> 1560 </span><span class="lineCov"> 37382 : l = nd - 1;</span></a>
|
|
<a name="1561"><span class="lineNum"> 1561 </span><span class="lineCov"> 37382 : p = v[nd];</span></a>
|
|
<a name="1562"><span class="lineNum"> 1562 </span><span class="lineCov"> 37382 : q = v[nd + 1];</span></a>
|
|
<a name="1563"><span class="lineNum"> 1563 </span><span class="lineCov"> 37382 : r = v[nd + 2];</span></a>
|
|
<a name="1564"><span class="lineNum"> 1564 </span><span class="lineCov"> 37382 : s = v[nd + 3];</span></a>
|
|
<a name="1565"><span class="lineNum"> 1565 </span><span class="lineCov"> 37382 : t = v[nd + 4];</span></a>
|
|
<a name="1566"><span class="lineNum"> 1566 </span> : </a>
|
|
<a name="1567"><span class="lineNum"> 1567 </span> : /* special case with only one data disk */</a>
|
|
<a name="1568"><span class="lineNum"> 1568 </span><span class="lineCov"> 37382 : if (l == 0) {</span></a>
|
|
<a name="1569"><span class="lineNum"> 1569 </span><span class="lineCov"> 6 : for (i = 0; i < 5; ++i)</span></a>
|
|
<a name="1570"><span class="lineNum"> 1570 </span><span class="lineCov"> 5 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1571"><span class="lineNum"> 1571 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1572"><span class="lineNum"> 1572 </span> : }</a>
|
|
<a name="1573"><span class="lineNum"> 1573 </span> : </a>
|
|
<a name="1574"><span class="lineNum"> 1574 </span> : raid_sse_begin();</a>
|
|
<a name="1575"><span class="lineNum"> 1575 </span> : </a>
|
|
<a name="1576"><span class="lineNum"> 1576 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1577"><span class="lineNum"> 1577 </span><span class="lineCov"> 37381 : asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1578"><span class="lineNum"> 1578 </span><span class="lineCov"> 37381 : asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1579"><span class="lineNum"> 1579 </span> : </a>
|
|
<a name="1580"><span class="lineNum"> 1580 </span><span class="lineCov"> 2625557 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="1581"><span class="lineNum"> 1581 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1582"><span class="lineNum"> 1582 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="1583"><span class="lineNum"> 1583 </span> : </a>
|
|
<a name="1584"><span class="lineNum"> 1584 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %xmm10,%xmm0");</span></a>
|
|
<a name="1585"><span class="lineNum"> 1585 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %xmm10,%xmm1");</span></a>
|
|
<a name="1586"><span class="lineNum"> 1586 </span> : </a>
|
|
<a name="1587"><span class="lineNum"> 1587 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %xmm10,%xmm11");</span></a>
|
|
<a name="1588"><span class="lineNum"> 1588 </span><span class="lineCov"> 2588176 : asm volatile ("psrlw $4,%xmm11");</span></a>
|
|
<a name="1589"><span class="lineNum"> 1589 </span><span class="lineCov"> 2588176 : asm volatile ("pand %xmm15,%xmm10");</span></a>
|
|
<a name="1590"><span class="lineNum"> 1590 </span><span class="lineCov"> 2588176 : asm volatile ("pand %xmm15,%xmm11");</span></a>
|
|
<a name="1591"><span class="lineNum"> 1591 </span> : </a>
|
|
<a name="1592"><span class="lineNum"> 1592 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1593"><span class="lineNum"> 1593 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1594"><span class="lineNum"> 1594 </span><span class="lineCov"> 2588176 : asm volatile ("pshufb %xmm10,%xmm2");</span></a>
|
|
<a name="1595"><span class="lineNum"> 1595 </span><span class="lineCov"> 2588176 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="1596"><span class="lineNum"> 1596 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm13,%xmm2");</span></a>
|
|
<a name="1597"><span class="lineNum"> 1597 </span> : </a>
|
|
<a name="1598"><span class="lineNum"> 1598 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1599"><span class="lineNum"> 1599 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1600"><span class="lineNum"> 1600 </span><span class="lineCov"> 2588176 : asm volatile ("pshufb %xmm10,%xmm3");</span></a>
|
|
<a name="1601"><span class="lineNum"> 1601 </span><span class="lineCov"> 2588176 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="1602"><span class="lineNum"> 1602 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm13,%xmm3");</span></a>
|
|
<a name="1603"><span class="lineNum"> 1603 </span> : </a>
|
|
<a name="1604"><span class="lineNum"> 1604 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="1605"><span class="lineNum"> 1605 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="1606"><span class="lineNum"> 1606 </span><span class="lineCov"> 2588176 : asm volatile ("pshufb %xmm10,%xmm4");</span></a>
|
|
<a name="1607"><span class="lineNum"> 1607 </span><span class="lineCov"> 2588176 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="1608"><span class="lineNum"> 1608 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm13,%xmm4");</span></a>
|
|
<a name="1609"><span class="lineNum"> 1609 </span> : </a>
|
|
<a name="1610"><span class="lineNum"> 1610 </span> : /* intermediate disks */</a>
|
|
<a name="1611"><span class="lineNum"> 1611 </span><span class="lineCov"> 13334512 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1612"><span class="lineNum"> 1612 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="1613"><span class="lineNum"> 1613 </span> : </a>
|
|
<a name="1614"><span class="lineNum"> 1614 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm11,%xmm11");</span></a>
|
|
<a name="1615"><span class="lineNum"> 1615 </span><span class="lineCov"> 10746336 : asm volatile ("pcmpgtb %xmm1,%xmm11");</span></a>
|
|
<a name="1616"><span class="lineNum"> 1616 </span><span class="lineCov"> 10746336 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="1617"><span class="lineNum"> 1617 </span><span class="lineCov"> 10746336 : asm volatile ("pand %xmm14,%xmm11");</span></a>
|
|
<a name="1618"><span class="lineNum"> 1618 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm11,%xmm1");</span></a>
|
|
<a name="1619"><span class="lineNum"> 1619 </span> : </a>
|
|
<a name="1620"><span class="lineNum"> 1620 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm10,%xmm0");</span></a>
|
|
<a name="1621"><span class="lineNum"> 1621 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm10,%xmm1");</span></a>
|
|
<a name="1622"><span class="lineNum"> 1622 </span> : </a>
|
|
<a name="1623"><span class="lineNum"> 1623 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %xmm10,%xmm11");</span></a>
|
|
<a name="1624"><span class="lineNum"> 1624 </span><span class="lineCov"> 10746336 : asm volatile ("psrlw $4,%xmm11");</span></a>
|
|
<a name="1625"><span class="lineNum"> 1625 </span><span class="lineCov"> 10746336 : asm volatile ("pand %xmm15,%xmm10");</span></a>
|
|
<a name="1626"><span class="lineNum"> 1626 </span><span class="lineCov"> 10746336 : asm volatile ("pand %xmm15,%xmm11");</span></a>
|
|
<a name="1627"><span class="lineNum"> 1627 </span> : </a>
|
|
<a name="1628"><span class="lineNum"> 1628 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1629"><span class="lineNum"> 1629 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1630"><span class="lineNum"> 1630 </span><span class="lineCov"> 10746336 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="1631"><span class="lineNum"> 1631 </span><span class="lineCov"> 10746336 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="1632"><span class="lineNum"> 1632 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm12,%xmm2");</span></a>
|
|
<a name="1633"><span class="lineNum"> 1633 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm13,%xmm2");</span></a>
|
|
<a name="1634"><span class="lineNum"> 1634 </span> : </a>
|
|
<a name="1635"><span class="lineNum"> 1635 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1636"><span class="lineNum"> 1636 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1637"><span class="lineNum"> 1637 </span><span class="lineCov"> 10746336 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="1638"><span class="lineNum"> 1638 </span><span class="lineCov"> 10746336 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="1639"><span class="lineNum"> 1639 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm12,%xmm3");</span></a>
|
|
<a name="1640"><span class="lineNum"> 1640 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm13,%xmm3");</span></a>
|
|
<a name="1641"><span class="lineNum"> 1641 </span> : </a>
|
|
<a name="1642"><span class="lineNum"> 1642 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="1643"><span class="lineNum"> 1643 </span><span class="lineCov"> 10746336 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="1644"><span class="lineNum"> 1644 </span><span class="lineCov"> 10746336 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="1645"><span class="lineNum"> 1645 </span><span class="lineCov"> 10746336 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="1646"><span class="lineNum"> 1646 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm12,%xmm4");</span></a>
|
|
<a name="1647"><span class="lineNum"> 1647 </span><span class="lineCov"> 10746336 : asm volatile ("pxor %xmm13,%xmm4");</span></a>
|
|
<a name="1648"><span class="lineNum"> 1648 </span> : }</a>
|
|
<a name="1649"><span class="lineNum"> 1649 </span> : </a>
|
|
<a name="1650"><span class="lineNum"> 1650 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1651"><span class="lineNum"> 1651 </span><span class="lineCov"> 2588176 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="1652"><span class="lineNum"> 1652 </span> : </a>
|
|
<a name="1653"><span class="lineNum"> 1653 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm11,%xmm11");</span></a>
|
|
<a name="1654"><span class="lineNum"> 1654 </span><span class="lineCov"> 2588176 : asm volatile ("pcmpgtb %xmm1,%xmm11");</span></a>
|
|
<a name="1655"><span class="lineNum"> 1655 </span><span class="lineCov"> 2588176 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="1656"><span class="lineNum"> 1656 </span><span class="lineCov"> 2588176 : asm volatile ("pand %xmm14,%xmm11");</span></a>
|
|
<a name="1657"><span class="lineNum"> 1657 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm11,%xmm1");</span></a>
|
|
<a name="1658"><span class="lineNum"> 1658 </span> : </a>
|
|
<a name="1659"><span class="lineNum"> 1659 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm10,%xmm0");</span></a>
|
|
<a name="1660"><span class="lineNum"> 1660 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm10,%xmm1");</span></a>
|
|
<a name="1661"><span class="lineNum"> 1661 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm10,%xmm2");</span></a>
|
|
<a name="1662"><span class="lineNum"> 1662 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm10,%xmm3");</span></a>
|
|
<a name="1663"><span class="lineNum"> 1663 </span><span class="lineCov"> 2588176 : asm volatile ("pxor %xmm10,%xmm4");</span></a>
|
|
<a name="1664"><span class="lineNum"> 1664 </span> : </a>
|
|
<a name="1665"><span class="lineNum"> 1665 </span><span class="lineCov"> 2588176 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1666"><span class="lineNum"> 1666 </span><span class="lineCov"> 2588176 : asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1667"><span class="lineNum"> 1667 </span><span class="lineCov"> 2588176 : asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1668"><span class="lineNum"> 1668 </span><span class="lineCov"> 2588176 : asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1669"><span class="lineNum"> 1669 </span><span class="lineCov"> 2588176 : asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i]));</span></a>
|
|
<a name="1670"><span class="lineNum"> 1670 </span> : }</a>
|
|
<a name="1671"><span class="lineNum"> 1671 </span> : </a>
|
|
<a name="1672"><span class="lineNum"> 1672 </span> : raid_sse_end();</a>
|
|
<a name="1673"><span class="lineNum"> 1673 </span> : }</a>
|
|
<a name="1674"><span class="lineNum"> 1674 </span> : #endif</a>
|
|
<a name="1675"><span class="lineNum"> 1675 </span> : </a>
|
|
<a name="1676"><span class="lineNum"> 1676 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX2)</a>
|
|
<a name="1677"><span class="lineNum"> 1677 </span> : /*</a>
|
|
<a name="1678"><span class="lineNum"> 1678 </span> : * GEN5 (penta parity with Cauchy matrix) AVX2 implementation</a>
|
|
<a name="1679"><span class="lineNum"> 1679 </span> : *</a>
|
|
<a name="1680"><span class="lineNum"> 1680 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="1681"><span class="lineNum"> 1681 </span> : */</a>
|
|
<a name="1682"><span class="lineNum"> 1682 </span><span class="lineCov"> 6 : void raid_gen5_avx2ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1683"><span class="lineNum"> 1683 </span> : {</a>
|
|
<a name="1684"><span class="lineNum"> 1684 </span><span class="lineCov"> 6 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1685"><span class="lineNum"> 1685 </span> : uint8_t *p;</a>
|
|
<a name="1686"><span class="lineNum"> 1686 </span> : uint8_t *q;</a>
|
|
<a name="1687"><span class="lineNum"> 1687 </span> : uint8_t *r;</a>
|
|
<a name="1688"><span class="lineNum"> 1688 </span> : uint8_t *s;</a>
|
|
<a name="1689"><span class="lineNum"> 1689 </span> : uint8_t *t;</a>
|
|
<a name="1690"><span class="lineNum"> 1690 </span> : int d, l;</a>
|
|
<a name="1691"><span class="lineNum"> 1691 </span> : size_t i;</a>
|
|
<a name="1692"><span class="lineNum"> 1692 </span> : </a>
|
|
<a name="1693"><span class="lineNum"> 1693 </span><span class="lineCov"> 6 : l = nd - 1;</span></a>
|
|
<a name="1694"><span class="lineNum"> 1694 </span><span class="lineCov"> 6 : p = v[nd];</span></a>
|
|
<a name="1695"><span class="lineNum"> 1695 </span><span class="lineCov"> 6 : q = v[nd + 1];</span></a>
|
|
<a name="1696"><span class="lineNum"> 1696 </span><span class="lineCov"> 6 : r = v[nd + 2];</span></a>
|
|
<a name="1697"><span class="lineNum"> 1697 </span><span class="lineCov"> 6 : s = v[nd + 3];</span></a>
|
|
<a name="1698"><span class="lineNum"> 1698 </span><span class="lineCov"> 6 : t = v[nd + 4];</span></a>
|
|
<a name="1699"><span class="lineNum"> 1699 </span> : </a>
|
|
<a name="1700"><span class="lineNum"> 1700 </span> : /* special case with only one data disk */</a>
|
|
<a name="1701"><span class="lineNum"> 1701 </span><span class="lineCov"> 6 : if (l == 0) {</span></a>
|
|
<a name="1702"><span class="lineNum"> 1702 </span><span class="lineCov"> 6 : for (i = 0; i < 5; ++i)</span></a>
|
|
<a name="1703"><span class="lineNum"> 1703 </span><span class="lineCov"> 5 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1704"><span class="lineNum"> 1704 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1705"><span class="lineNum"> 1705 </span> : }</a>
|
|
<a name="1706"><span class="lineNum"> 1706 </span> : </a>
|
|
<a name="1707"><span class="lineNum"> 1707 </span> : raid_avx_begin();</a>
|
|
<a name="1708"><span class="lineNum"> 1708 </span> : </a>
|
|
<a name="1709"><span class="lineNum"> 1709 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1710"><span class="lineNum"> 1710 </span><span class="lineCov"> 5 : asm volatile ("vpxor %ymm6,%ymm6,%ymm6");</span></a>
|
|
<a name="1711"><span class="lineNum"> 1711 </span><span class="lineCov"> 5 : asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1712"><span class="lineNum"> 1712 </span><span class="lineCov"> 5 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1713"><span class="lineNum"> 1713 </span> : </a>
|
|
<a name="1714"><span class="lineNum"> 1714 </span><span class="lineCov"> 32781 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="1715"><span class="lineNum"> 1715 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1716"><span class="lineNum"> 1716 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="1717"><span class="lineNum"> 1717 </span> : </a>
|
|
<a name="1718"><span class="lineNum"> 1718 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %ymm10,%ymm0");</span></a>
|
|
<a name="1719"><span class="lineNum"> 1719 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %ymm10,%ymm1");</span></a>
|
|
<a name="1720"><span class="lineNum"> 1720 </span> : </a>
|
|
<a name="1721"><span class="lineNum"> 1721 </span><span class="lineCov"> 32776 : asm volatile ("vpsrlw $4,%ymm10,%ymm11");</span></a>
|
|
<a name="1722"><span class="lineNum"> 1722 </span><span class="lineCov"> 32776 : asm volatile ("vpand %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="1723"><span class="lineNum"> 1723 </span><span class="lineCov"> 32776 : asm volatile ("vpand %ymm15,%ymm11,%ymm11");</span></a>
|
|
<a name="1724"><span class="lineNum"> 1724 </span> : </a>
|
|
<a name="1725"><span class="lineNum"> 1725 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1726"><span class="lineNum"> 1726 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1727"><span class="lineNum"> 1727 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm2,%ymm2");</span></a>
|
|
<a name="1728"><span class="lineNum"> 1728 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="1729"><span class="lineNum"> 1729 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm2,%ymm2");</span></a>
|
|
<a name="1730"><span class="lineNum"> 1730 </span> : </a>
|
|
<a name="1731"><span class="lineNum"> 1731 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1732"><span class="lineNum"> 1732 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1733"><span class="lineNum"> 1733 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm3,%ymm3");</span></a>
|
|
<a name="1734"><span class="lineNum"> 1734 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="1735"><span class="lineNum"> 1735 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm3,%ymm3");</span></a>
|
|
<a name="1736"><span class="lineNum"> 1736 </span> : </a>
|
|
<a name="1737"><span class="lineNum"> 1737 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="1738"><span class="lineNum"> 1738 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="1739"><span class="lineNum"> 1739 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm4,%ymm4");</span></a>
|
|
<a name="1740"><span class="lineNum"> 1740 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="1741"><span class="lineNum"> 1741 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm4,%ymm4");</span></a>
|
|
<a name="1742"><span class="lineNum"> 1742 </span> : </a>
|
|
<a name="1743"><span class="lineNum"> 1743 </span> : /* intermediate disks */</a>
|
|
<a name="1744"><span class="lineNum"> 1744 </span><span class="lineCov"> 229624 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1745"><span class="lineNum"> 1745 </span><span class="lineCov"> 196848 : asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="1746"><span class="lineNum"> 1746 </span> : </a>
|
|
<a name="1747"><span class="lineNum"> 1747 </span><span class="lineCov"> 196848 : asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");</span></a>
|
|
<a name="1748"><span class="lineNum"> 1748 </span><span class="lineCov"> 196848 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="1749"><span class="lineNum"> 1749 </span><span class="lineCov"> 196848 : asm volatile ("vpand %ymm14,%ymm11,%ymm11");</span></a>
|
|
<a name="1750"><span class="lineNum"> 1750 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm11,%ymm1,%ymm1");</span></a>
|
|
<a name="1751"><span class="lineNum"> 1751 </span> : </a>
|
|
<a name="1752"><span class="lineNum"> 1752 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm10,%ymm0,%ymm0");</span></a>
|
|
<a name="1753"><span class="lineNum"> 1753 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm10,%ymm1,%ymm1");</span></a>
|
|
<a name="1754"><span class="lineNum"> 1754 </span> : </a>
|
|
<a name="1755"><span class="lineNum"> 1755 </span><span class="lineCov"> 196848 : asm volatile ("vpsrlw $4,%ymm10,%ymm11");</span></a>
|
|
<a name="1756"><span class="lineNum"> 1756 </span><span class="lineCov"> 196848 : asm volatile ("vpand %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="1757"><span class="lineNum"> 1757 </span><span class="lineCov"> 196848 : asm volatile ("vpand %ymm15,%ymm11,%ymm11");</span></a>
|
|
<a name="1758"><span class="lineNum"> 1758 </span> : </a>
|
|
<a name="1759"><span class="lineNum"> 1759 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1760"><span class="lineNum"> 1760 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1761"><span class="lineNum"> 1761 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="1762"><span class="lineNum"> 1762 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="1763"><span class="lineNum"> 1763 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm2,%ymm2");</span></a>
|
|
<a name="1764"><span class="lineNum"> 1764 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm2,%ymm2");</span></a>
|
|
<a name="1765"><span class="lineNum"> 1765 </span> : </a>
|
|
<a name="1766"><span class="lineNum"> 1766 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1767"><span class="lineNum"> 1767 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1768"><span class="lineNum"> 1768 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="1769"><span class="lineNum"> 1769 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="1770"><span class="lineNum"> 1770 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm3,%ymm3");</span></a>
|
|
<a name="1771"><span class="lineNum"> 1771 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm3,%ymm3");</span></a>
|
|
<a name="1772"><span class="lineNum"> 1772 </span> : </a>
|
|
<a name="1773"><span class="lineNum"> 1773 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="1774"><span class="lineNum"> 1774 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="1775"><span class="lineNum"> 1775 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="1776"><span class="lineNum"> 1776 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="1777"><span class="lineNum"> 1777 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm4,%ymm4");</span></a>
|
|
<a name="1778"><span class="lineNum"> 1778 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm4,%ymm4");</span></a>
|
|
<a name="1779"><span class="lineNum"> 1779 </span> : }</a>
|
|
<a name="1780"><span class="lineNum"> 1780 </span> : </a>
|
|
<a name="1781"><span class="lineNum"> 1781 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1782"><span class="lineNum"> 1782 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="1783"><span class="lineNum"> 1783 </span> : </a>
|
|
<a name="1784"><span class="lineNum"> 1784 </span><span class="lineCov"> 32776 : asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");</span></a>
|
|
<a name="1785"><span class="lineNum"> 1785 </span><span class="lineCov"> 32776 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="1786"><span class="lineNum"> 1786 </span><span class="lineCov"> 32776 : asm volatile ("vpand %ymm14,%ymm11,%ymm11");</span></a>
|
|
<a name="1787"><span class="lineNum"> 1787 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm11,%ymm1,%ymm1");</span></a>
|
|
<a name="1788"><span class="lineNum"> 1788 </span> : </a>
|
|
<a name="1789"><span class="lineNum"> 1789 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm0,%ymm0");</span></a>
|
|
<a name="1790"><span class="lineNum"> 1790 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm1,%ymm1");</span></a>
|
|
<a name="1791"><span class="lineNum"> 1791 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm2,%ymm2");</span></a>
|
|
<a name="1792"><span class="lineNum"> 1792 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm3,%ymm3");</span></a>
|
|
<a name="1793"><span class="lineNum"> 1793 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm4,%ymm4");</span></a>
|
|
<a name="1794"><span class="lineNum"> 1794 </span> : </a>
|
|
<a name="1795"><span class="lineNum"> 1795 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1796"><span class="lineNum"> 1796 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1797"><span class="lineNum"> 1797 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1798"><span class="lineNum"> 1798 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1799"><span class="lineNum"> 1799 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i]));</span></a>
|
|
<a name="1800"><span class="lineNum"> 1800 </span> : }</a>
|
|
<a name="1801"><span class="lineNum"> 1801 </span> : </a>
|
|
<a name="1802"><span class="lineNum"> 1802 </span> : raid_avx_end();</a>
|
|
<a name="1803"><span class="lineNum"> 1803 </span> : }</a>
|
|
<a name="1804"><span class="lineNum"> 1804 </span> : #endif</a>
|
|
<a name="1805"><span class="lineNum"> 1805 </span> : </a>
|
|
<a name="1806"><span class="lineNum"> 1806 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="1807"><span class="lineNum"> 1807 </span> : /*</a>
|
|
<a name="1808"><span class="lineNum"> 1808 </span> : * GEN5 (penta parity with Cauchy matrix) AVX512BW implementation</a>
|
|
<a name="1809"><span class="lineNum"> 1809 </span> : */</a>
|
|
<a name="1810"><span class="lineNum"> 1810 </span><span class="lineCov"> 17482 : void raid_gen5_avx512bw(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1811"><span class="lineNum"> 1811 </span> : {</a>
|
|
<a name="1812"><span class="lineNum"> 1812 </span><span class="lineCov"> 17482 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1813"><span class="lineNum"> 1813 </span> : uint8_t *p;</a>
|
|
<a name="1814"><span class="lineNum"> 1814 </span> : uint8_t *q;</a>
|
|
<a name="1815"><span class="lineNum"> 1815 </span> : uint8_t *r;</a>
|
|
<a name="1816"><span class="lineNum"> 1816 </span> : uint8_t *s;</a>
|
|
<a name="1817"><span class="lineNum"> 1817 </span> : uint8_t *t;</a>
|
|
<a name="1818"><span class="lineNum"> 1818 </span> : int d, l;</a>
|
|
<a name="1819"><span class="lineNum"> 1819 </span> : size_t i;</a>
|
|
<a name="1820"><span class="lineNum"> 1820 </span> : </a>
|
|
<a name="1821"><span class="lineNum"> 1821 </span><span class="lineCov"> 17482 : l = nd - 1;</span></a>
|
|
<a name="1822"><span class="lineNum"> 1822 </span><span class="lineCov"> 17482 : p = v[nd];</span></a>
|
|
<a name="1823"><span class="lineNum"> 1823 </span><span class="lineCov"> 17482 : q = v[nd + 1];</span></a>
|
|
<a name="1824"><span class="lineNum"> 1824 </span><span class="lineCov"> 17482 : r = v[nd + 2];</span></a>
|
|
<a name="1825"><span class="lineNum"> 1825 </span><span class="lineCov"> 17482 : s = v[nd + 3];</span></a>
|
|
<a name="1826"><span class="lineNum"> 1826 </span><span class="lineCov"> 17482 : t = v[nd + 4];</span></a>
|
|
<a name="1827"><span class="lineNum"> 1827 </span> : </a>
|
|
<a name="1828"><span class="lineNum"> 1828 </span> : /* special case with only one data disk */</a>
|
|
<a name="1829"><span class="lineNum"> 1829 </span><span class="lineCov"> 17482 : if (l == 0) {</span></a>
|
|
<a name="1830"><span class="lineNum"> 1830 </span><span class="lineCov"> 6 : for (i = 0; i < 5; ++i)</span></a>
|
|
<a name="1831"><span class="lineNum"> 1831 </span><span class="lineCov"> 5 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1832"><span class="lineNum"> 1832 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1833"><span class="lineNum"> 1833 </span> : }</a>
|
|
<a name="1834"><span class="lineNum"> 1834 </span> : </a>
|
|
<a name="1835"><span class="lineNum"> 1835 </span> : raid_avx_begin();</a>
|
|
<a name="1836"><span class="lineNum"> 1836 </span> : </a>
|
|
<a name="1837"><span class="lineNum"> 1837 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1838"><span class="lineNum"> 1838 </span><span class="lineCov"> 17481 : asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="1839"><span class="lineNum"> 1839 </span><span class="lineCov"> 17481 : asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="1840"><span class="lineNum"> 1840 </span><span class="lineCov"> 17481 : asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1841"><span class="lineNum"> 1841 </span> : </a>
|
|
<a name="1842"><span class="lineNum"> 1842 </span><span class="lineCov"> 95949 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="1843"><span class="lineNum"> 1843 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1844"><span class="lineNum"> 1844 </span><span class="lineCov"> 78468 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="1845"><span class="lineNum"> 1845 </span> : </a>
|
|
<a name="1846"><span class="lineNum"> 1846 </span><span class="lineCov"> 78468 : asm volatile ("vmovdqa64 %zmm10,%zmm0");</span></a>
|
|
<a name="1847"><span class="lineNum"> 1847 </span><span class="lineCov"> 78468 : asm volatile ("vmovdqa64 %zmm10,%zmm1");</span></a>
|
|
<a name="1848"><span class="lineNum"> 1848 </span> : </a>
|
|
<a name="1849"><span class="lineNum"> 1849 </span><span class="lineCov"> 78468 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="1850"><span class="lineNum"> 1850 </span><span class="lineCov"> 78468 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="1851"><span class="lineNum"> 1851 </span><span class="lineCov"> 78468 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="1852"><span class="lineNum"> 1852 </span> : </a>
|
|
<a name="1853"><span class="lineNum"> 1853 </span><span class="lineCov"> 78468 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1854"><span class="lineNum"> 1854 </span><span class="lineCov"> 78468 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1855"><span class="lineNum"> 1855 </span><span class="lineCov"> 78468 : asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="1856"><span class="lineNum"> 1856 </span><span class="lineCov"> 78468 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1857"><span class="lineNum"> 1857 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="1858"><span class="lineNum"> 1858 </span> : </a>
|
|
<a name="1859"><span class="lineNum"> 1859 </span><span class="lineCov"> 78468 : asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1860"><span class="lineNum"> 1860 </span><span class="lineCov"> 78468 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1861"><span class="lineNum"> 1861 </span><span class="lineCov"> 78468 : asm volatile ("vpshufb %zmm10,%zmm3,%zmm3");</span></a>
|
|
<a name="1862"><span class="lineNum"> 1862 </span><span class="lineCov"> 78468 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1863"><span class="lineNum"> 1863 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");</span></a>
|
|
<a name="1864"><span class="lineNum"> 1864 </span> : </a>
|
|
<a name="1865"><span class="lineNum"> 1865 </span><span class="lineCov"> 78468 : asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="1866"><span class="lineNum"> 1866 </span><span class="lineCov"> 78468 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="1867"><span class="lineNum"> 1867 </span><span class="lineCov"> 78468 : asm volatile ("vpshufb %zmm10,%zmm4,%zmm4");</span></a>
|
|
<a name="1868"><span class="lineNum"> 1868 </span><span class="lineCov"> 78468 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1869"><span class="lineNum"> 1869 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");</span></a>
|
|
<a name="1870"><span class="lineNum"> 1870 </span> : </a>
|
|
<a name="1871"><span class="lineNum"> 1871 </span> : /* intermediate disks */</a>
|
|
<a name="1872"><span class="lineNum"> 1872 </span><span class="lineCov"> 831996 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="1873"><span class="lineNum"> 1873 </span><span class="lineCov"> 753528 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="1874"><span class="lineNum"> 1874 </span> : </a>
|
|
<a name="1875"><span class="lineNum"> 1875 </span><span class="lineCov"> 753528 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="1876"><span class="lineNum"> 1876 </span><span class="lineCov"> 753528 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="1877"><span class="lineNum"> 1877 </span><span class="lineCov"> 753528 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="1878"><span class="lineNum"> 1878 </span><span class="lineCov"> 753528 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="1879"><span class="lineNum"> 1879 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="1880"><span class="lineNum"> 1880 </span> : </a>
|
|
<a name="1881"><span class="lineNum"> 1881 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="1882"><span class="lineNum"> 1882 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="1883"><span class="lineNum"> 1883 </span> : </a>
|
|
<a name="1884"><span class="lineNum"> 1884 </span><span class="lineCov"> 753528 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="1885"><span class="lineNum"> 1885 </span><span class="lineCov"> 753528 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="1886"><span class="lineNum"> 1886 </span><span class="lineCov"> 753528 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="1887"><span class="lineNum"> 1887 </span> : </a>
|
|
<a name="1888"><span class="lineNum"> 1888 </span><span class="lineCov"> 753528 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="1889"><span class="lineNum"> 1889 </span><span class="lineCov"> 753528 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="1890"><span class="lineNum"> 1890 </span><span class="lineCov"> 753528 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="1891"><span class="lineNum"> 1891 </span><span class="lineCov"> 753528 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1892"><span class="lineNum"> 1892 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");</span></a>
|
|
<a name="1893"><span class="lineNum"> 1893 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="1894"><span class="lineNum"> 1894 </span> : </a>
|
|
<a name="1895"><span class="lineNum"> 1895 </span><span class="lineCov"> 753528 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="1896"><span class="lineNum"> 1896 </span><span class="lineCov"> 753528 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="1897"><span class="lineNum"> 1897 </span><span class="lineCov"> 753528 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="1898"><span class="lineNum"> 1898 </span><span class="lineCov"> 753528 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1899"><span class="lineNum"> 1899 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm12,%zmm3,%zmm3");</span></a>
|
|
<a name="1900"><span class="lineNum"> 1900 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");</span></a>
|
|
<a name="1901"><span class="lineNum"> 1901 </span> : </a>
|
|
<a name="1902"><span class="lineNum"> 1902 </span><span class="lineCov"> 753528 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="1903"><span class="lineNum"> 1903 </span><span class="lineCov"> 753528 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="1904"><span class="lineNum"> 1904 </span><span class="lineCov"> 753528 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="1905"><span class="lineNum"> 1905 </span><span class="lineCov"> 753528 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="1906"><span class="lineNum"> 1906 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm12,%zmm4,%zmm4");</span></a>
|
|
<a name="1907"><span class="lineNum"> 1907 </span><span class="lineCov"> 753528 : asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");</span></a>
|
|
<a name="1908"><span class="lineNum"> 1908 </span> : }</a>
|
|
<a name="1909"><span class="lineNum"> 1909 </span> : </a>
|
|
<a name="1910"><span class="lineNum"> 1910 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="1911"><span class="lineNum"> 1911 </span><span class="lineCov"> 78468 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="1912"><span class="lineNum"> 1912 </span> : </a>
|
|
<a name="1913"><span class="lineNum"> 1913 </span><span class="lineCov"> 78468 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="1914"><span class="lineNum"> 1914 </span><span class="lineCov"> 78468 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="1915"><span class="lineNum"> 1915 </span><span class="lineCov"> 78468 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="1916"><span class="lineNum"> 1916 </span><span class="lineCov"> 78468 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="1917"><span class="lineNum"> 1917 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="1918"><span class="lineNum"> 1918 </span> : </a>
|
|
<a name="1919"><span class="lineNum"> 1919 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="1920"><span class="lineNum"> 1920 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="1921"><span class="lineNum"> 1921 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="1922"><span class="lineNum"> 1922 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm10,%zmm3,%zmm3");</span></a>
|
|
<a name="1923"><span class="lineNum"> 1923 </span><span class="lineCov"> 78468 : asm volatile ("vpxorq %zmm10,%zmm4,%zmm4");</span></a>
|
|
<a name="1924"><span class="lineNum"> 1924 </span> : </a>
|
|
<a name="1925"><span class="lineNum"> 1925 </span><span class="lineCov"> 78468 : asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="1926"><span class="lineNum"> 1926 </span><span class="lineCov"> 78468 : asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="1927"><span class="lineNum"> 1927 </span><span class="lineCov"> 78468 : asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="1928"><span class="lineNum"> 1928 </span><span class="lineCov"> 78468 : asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="1929"><span class="lineNum"> 1929 </span><span class="lineCov"> 78468 : asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i]));</span></a>
|
|
<a name="1930"><span class="lineNum"> 1930 </span> : }</a>
|
|
<a name="1931"><span class="lineNum"> 1931 </span> : </a>
|
|
<a name="1932"><span class="lineNum"> 1932 </span> : raid_avx_end();</a>
|
|
<a name="1933"><span class="lineNum"> 1933 </span> : }</a>
|
|
<a name="1934"><span class="lineNum"> 1934 </span> : #endif</a>
|
|
<a name="1935"><span class="lineNum"> 1935 </span> : </a>
|
|
<a name="1936"><span class="lineNum"> 1936 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="1937"><span class="lineNum"> 1937 </span> : /*</a>
|
|
<a name="1938"><span class="lineNum"> 1938 </span> : * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="1939"><span class="lineNum"> 1939 </span> : */</a>
|
|
<a name="1940"><span class="lineNum"> 1940 </span><span class="lineCov"> 10 : void raid_gen6_ssse3(int nd, size_t size, void **vv)</span></a>
|
|
<a name="1941"><span class="lineNum"> 1941 </span> : {</a>
|
|
<a name="1942"><span class="lineNum"> 1942 </span><span class="lineCov"> 10 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="1943"><span class="lineNum"> 1943 </span> : uint8_t *p;</a>
|
|
<a name="1944"><span class="lineNum"> 1944 </span> : uint8_t *q;</a>
|
|
<a name="1945"><span class="lineNum"> 1945 </span> : uint8_t *r;</a>
|
|
<a name="1946"><span class="lineNum"> 1946 </span> : uint8_t *s;</a>
|
|
<a name="1947"><span class="lineNum"> 1947 </span> : uint8_t *t;</a>
|
|
<a name="1948"><span class="lineNum"> 1948 </span> : uint8_t *u;</a>
|
|
<a name="1949"><span class="lineNum"> 1949 </span> : int d, l;</a>
|
|
<a name="1950"><span class="lineNum"> 1950 </span> : size_t i;</a>
|
|
<a name="1951"><span class="lineNum"> 1951 </span> : uint8_t buffer[2*16+16];</a>
|
|
<a name="1952"><span class="lineNum"> 1952 </span><span class="lineCov"> 10 : uint8_t *pd = __align_ptr(buffer, 16);</span></a>
|
|
<a name="1953"><span class="lineNum"> 1953 </span> : </a>
|
|
<a name="1954"><span class="lineNum"> 1954 </span><span class="lineCov"> 10 : l = nd - 1;</span></a>
|
|
<a name="1955"><span class="lineNum"> 1955 </span><span class="lineCov"> 10 : p = v[nd];</span></a>
|
|
<a name="1956"><span class="lineNum"> 1956 </span><span class="lineCov"> 10 : q = v[nd + 1];</span></a>
|
|
<a name="1957"><span class="lineNum"> 1957 </span><span class="lineCov"> 10 : r = v[nd + 2];</span></a>
|
|
<a name="1958"><span class="lineNum"> 1958 </span><span class="lineCov"> 10 : s = v[nd + 3];</span></a>
|
|
<a name="1959"><span class="lineNum"> 1959 </span><span class="lineCov"> 10 : t = v[nd + 4];</span></a>
|
|
<a name="1960"><span class="lineNum"> 1960 </span><span class="lineCov"> 10 : u = v[nd + 5];</span></a>
|
|
<a name="1961"><span class="lineNum"> 1961 </span> : </a>
|
|
<a name="1962"><span class="lineNum"> 1962 </span> : /* special case with only one data disk */</a>
|
|
<a name="1963"><span class="lineNum"> 1963 </span><span class="lineCov"> 10 : if (l == 0) {</span></a>
|
|
<a name="1964"><span class="lineNum"> 1964 </span><span class="lineCov"> 7 : for (i = 0; i < 6; ++i)</span></a>
|
|
<a name="1965"><span class="lineNum"> 1965 </span><span class="lineCov"> 6 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="1966"><span class="lineNum"> 1966 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="1967"><span class="lineNum"> 1967 </span> : }</a>
|
|
<a name="1968"><span class="lineNum"> 1968 </span> : </a>
|
|
<a name="1969"><span class="lineNum"> 1969 </span> : raid_sse_begin();</a>
|
|
<a name="1970"><span class="lineNum"> 1970 </span> : </a>
|
|
<a name="1971"><span class="lineNum"> 1971 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="1972"><span class="lineNum"> 1972 </span><span class="lineCov"> 131097 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="1973"><span class="lineNum"> 1973 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="1974"><span class="lineNum"> 1974 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));</span></a>
|
|
<a name="1975"><span class="lineNum"> 1975 </span> : </a>
|
|
<a name="1976"><span class="lineNum"> 1976 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));</span></a>
|
|
<a name="1977"><span class="lineNum"> 1977 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16]));</span></a>
|
|
<a name="1978"><span class="lineNum"> 1978 </span> : </a>
|
|
<a name="1979"><span class="lineNum"> 1979 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="1980"><span class="lineNum"> 1980 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="1981"><span class="lineNum"> 1981 </span><span class="lineCov"> 131088 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="1982"><span class="lineNum"> 1982 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="1983"><span class="lineNum"> 1983 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="1984"><span class="lineNum"> 1984 </span> : </a>
|
|
<a name="1985"><span class="lineNum"> 1985 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm0" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="1986"><span class="lineNum"> 1986 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="1987"><span class="lineNum"> 1987 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm0");</span></a>
|
|
<a name="1988"><span class="lineNum"> 1988 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1989"><span class="lineNum"> 1989 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm0");</span></a>
|
|
<a name="1990"><span class="lineNum"> 1990 </span> : </a>
|
|
<a name="1991"><span class="lineNum"> 1991 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm1" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="1992"><span class="lineNum"> 1992 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="1993"><span class="lineNum"> 1993 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm1");</span></a>
|
|
<a name="1994"><span class="lineNum"> 1994 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="1995"><span class="lineNum"> 1995 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm1");</span></a>
|
|
<a name="1996"><span class="lineNum"> 1996 </span> : </a>
|
|
<a name="1997"><span class="lineNum"> 1997 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="1998"><span class="lineNum"> 1998 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="1999"><span class="lineNum"> 1999 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="2000"><span class="lineNum"> 2000 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="2001"><span class="lineNum"> 2001 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="2002"><span class="lineNum"> 2002 </span> : </a>
|
|
<a name="2003"><span class="lineNum"> 2003 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][3][0][0]));</span></a>
|
|
<a name="2004"><span class="lineNum"> 2004 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[l][3][1][0]));</span></a>
|
|
<a name="2005"><span class="lineNum"> 2005 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm4,%xmm3");</span></a>
|
|
<a name="2006"><span class="lineNum"> 2006 </span><span class="lineCov"> 131088 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="2007"><span class="lineNum"> 2007 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="2008"><span class="lineNum"> 2008 </span> : </a>
|
|
<a name="2009"><span class="lineNum"> 2009 </span> : /* intermediate disks */</a>
|
|
<a name="2010"><span class="lineNum"> 2010 </span><span class="lineCov"> 918000 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="2011"><span class="lineNum"> 2011 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));</span></a>
|
|
<a name="2012"><span class="lineNum"> 2012 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));</span></a>
|
|
<a name="2013"><span class="lineNum"> 2013 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="2014"><span class="lineNum"> 2014 </span> : </a>
|
|
<a name="2015"><span class="lineNum"> 2015 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm4");</span></a>
|
|
<a name="2016"><span class="lineNum"> 2016 </span><span class="lineCov"> 786912 : asm volatile ("pcmpgtb %xmm6,%xmm4");</span></a>
|
|
<a name="2017"><span class="lineNum"> 2017 </span><span class="lineCov"> 786912 : asm volatile ("paddb %xmm6,%xmm6");</span></a>
|
|
<a name="2018"><span class="lineNum"> 2018 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2019"><span class="lineNum"> 2019 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm6");</span></a>
|
|
<a name="2020"><span class="lineNum"> 2020 </span> : </a>
|
|
<a name="2021"><span class="lineNum"> 2021 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));</span></a>
|
|
<a name="2022"><span class="lineNum"> 2022 </span> : </a>
|
|
<a name="2023"><span class="lineNum"> 2023 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm5");</span></a>
|
|
<a name="2024"><span class="lineNum"> 2024 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm4,%xmm6");</span></a>
|
|
<a name="2025"><span class="lineNum"> 2025 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0]));</span></a>
|
|
<a name="2026"><span class="lineNum"> 2026 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16]));</span></a>
|
|
<a name="2027"><span class="lineNum"> 2027 </span> : </a>
|
|
<a name="2028"><span class="lineNum"> 2028 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2029"><span class="lineNum"> 2029 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="2030"><span class="lineNum"> 2030 </span><span class="lineCov"> 786912 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="2031"><span class="lineNum"> 2031 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2032"><span class="lineNum"> 2032 </span><span class="lineCov"> 786912 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="2033"><span class="lineNum"> 2033 </span> : </a>
|
|
<a name="2034"><span class="lineNum"> 2034 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="2035"><span class="lineNum"> 2035 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="2036"><span class="lineNum"> 2036 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="2037"><span class="lineNum"> 2037 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="2038"><span class="lineNum"> 2038 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm0");</span></a>
|
|
<a name="2039"><span class="lineNum"> 2039 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm0");</span></a>
|
|
<a name="2040"><span class="lineNum"> 2040 </span> : </a>
|
|
<a name="2041"><span class="lineNum"> 2041 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="2042"><span class="lineNum"> 2042 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="2043"><span class="lineNum"> 2043 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="2044"><span class="lineNum"> 2044 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="2045"><span class="lineNum"> 2045 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm1");</span></a>
|
|
<a name="2046"><span class="lineNum"> 2046 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm1");</span></a>
|
|
<a name="2047"><span class="lineNum"> 2047 </span> : </a>
|
|
<a name="2048"><span class="lineNum"> 2048 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="2049"><span class="lineNum"> 2049 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="2050"><span class="lineNum"> 2050 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="2051"><span class="lineNum"> 2051 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="2052"><span class="lineNum"> 2052 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm2");</span></a>
|
|
<a name="2053"><span class="lineNum"> 2053 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm2");</span></a>
|
|
<a name="2054"><span class="lineNum"> 2054 </span> : </a>
|
|
<a name="2055"><span class="lineNum"> 2055 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][3][0][0]));</span></a>
|
|
<a name="2056"><span class="lineNum"> 2056 </span><span class="lineCov"> 786912 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][3][1][0]));</span></a>
|
|
<a name="2057"><span class="lineNum"> 2057 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm4,%xmm6");</span></a>
|
|
<a name="2058"><span class="lineNum"> 2058 </span><span class="lineCov"> 786912 : asm volatile ("pshufb %xmm5,%xmm7");</span></a>
|
|
<a name="2059"><span class="lineNum"> 2059 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm6,%xmm3");</span></a>
|
|
<a name="2060"><span class="lineNum"> 2060 </span><span class="lineCov"> 786912 : asm volatile ("pxor %xmm7,%xmm3");</span></a>
|
|
<a name="2061"><span class="lineNum"> 2061 </span> : }</a>
|
|
<a name="2062"><span class="lineNum"> 2062 </span> : </a>
|
|
<a name="2063"><span class="lineNum"> 2063 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="2064"><span class="lineNum"> 2064 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));</span></a>
|
|
<a name="2065"><span class="lineNum"> 2065 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));</span></a>
|
|
<a name="2066"><span class="lineNum"> 2066 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="2067"><span class="lineNum"> 2067 </span> : </a>
|
|
<a name="2068"><span class="lineNum"> 2068 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm4");</span></a>
|
|
<a name="2069"><span class="lineNum"> 2069 </span><span class="lineCov"> 131088 : asm volatile ("pcmpgtb %xmm6,%xmm4");</span></a>
|
|
<a name="2070"><span class="lineNum"> 2070 </span><span class="lineCov"> 131088 : asm volatile ("paddb %xmm6,%xmm6");</span></a>
|
|
<a name="2071"><span class="lineNum"> 2071 </span><span class="lineCov"> 131088 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2072"><span class="lineNum"> 2072 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm6");</span></a>
|
|
<a name="2073"><span class="lineNum"> 2073 </span> : </a>
|
|
<a name="2074"><span class="lineNum"> 2074 </span><span class="lineCov"> 131088 : asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));</span></a>
|
|
<a name="2075"><span class="lineNum"> 2075 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm0");</span></a>
|
|
<a name="2076"><span class="lineNum"> 2076 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm1");</span></a>
|
|
<a name="2077"><span class="lineNum"> 2077 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm2");</span></a>
|
|
<a name="2078"><span class="lineNum"> 2078 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm3");</span></a>
|
|
<a name="2079"><span class="lineNum"> 2079 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm5");</span></a>
|
|
<a name="2080"><span class="lineNum"> 2080 </span><span class="lineCov"> 131088 : asm volatile ("pxor %xmm4,%xmm6");</span></a>
|
|
<a name="2081"><span class="lineNum"> 2081 </span> : </a>
|
|
<a name="2082"><span class="lineNum"> 2082 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i]));</span></a>
|
|
<a name="2083"><span class="lineNum"> 2083 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i]));</span></a>
|
|
<a name="2084"><span class="lineNum"> 2084 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i]));</span></a>
|
|
<a name="2085"><span class="lineNum"> 2085 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i]));</span></a>
|
|
<a name="2086"><span class="lineNum"> 2086 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i]));</span></a>
|
|
<a name="2087"><span class="lineNum"> 2087 </span><span class="lineCov"> 131088 : asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i]));</span></a>
|
|
<a name="2088"><span class="lineNum"> 2088 </span> : }</a>
|
|
<a name="2089"><span class="lineNum"> 2089 </span> : </a>
|
|
<a name="2090"><span class="lineNum"> 2090 </span> : raid_sse_end();</a>
|
|
<a name="2091"><span class="lineNum"> 2091 </span> : }</a>
|
|
<a name="2092"><span class="lineNum"> 2092 </span> : #endif</a>
|
|
<a name="2093"><span class="lineNum"> 2093 </span> : </a>
|
|
<a name="2094"><span class="lineNum"> 2094 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_SSSE3)</a>
|
|
<a name="2095"><span class="lineNum"> 2095 </span> : /*</a>
|
|
<a name="2096"><span class="lineNum"> 2096 </span> : * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation</a>
|
|
<a name="2097"><span class="lineNum"> 2097 </span> : *</a>
|
|
<a name="2098"><span class="lineNum"> 2098 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="2099"><span class="lineNum"> 2099 </span> : */</a>
|
|
<a name="2100"><span class="lineNum"> 2100 </span><span class="lineCov"> 539878 : void raid_gen6_ssse3ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="2101"><span class="lineNum"> 2101 </span> : {</a>
|
|
<a name="2102"><span class="lineNum"> 2102 </span><span class="lineCov"> 539878 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2103"><span class="lineNum"> 2103 </span> : uint8_t *p;</a>
|
|
<a name="2104"><span class="lineNum"> 2104 </span> : uint8_t *q;</a>
|
|
<a name="2105"><span class="lineNum"> 2105 </span> : uint8_t *r;</a>
|
|
<a name="2106"><span class="lineNum"> 2106 </span> : uint8_t *s;</a>
|
|
<a name="2107"><span class="lineNum"> 2107 </span> : uint8_t *t;</a>
|
|
<a name="2108"><span class="lineNum"> 2108 </span> : uint8_t *u;</a>
|
|
<a name="2109"><span class="lineNum"> 2109 </span> : int d, l;</a>
|
|
<a name="2110"><span class="lineNum"> 2110 </span> : size_t i;</a>
|
|
<a name="2111"><span class="lineNum"> 2111 </span> : </a>
|
|
<a name="2112"><span class="lineNum"> 2112 </span><span class="lineCov"> 539878 : l = nd - 1;</span></a>
|
|
<a name="2113"><span class="lineNum"> 2113 </span><span class="lineCov"> 539878 : p = v[nd];</span></a>
|
|
<a name="2114"><span class="lineNum"> 2114 </span><span class="lineCov"> 539878 : q = v[nd + 1];</span></a>
|
|
<a name="2115"><span class="lineNum"> 2115 </span><span class="lineCov"> 539878 : r = v[nd + 2];</span></a>
|
|
<a name="2116"><span class="lineNum"> 2116 </span><span class="lineCov"> 539878 : s = v[nd + 3];</span></a>
|
|
<a name="2117"><span class="lineNum"> 2117 </span><span class="lineCov"> 539878 : t = v[nd + 4];</span></a>
|
|
<a name="2118"><span class="lineNum"> 2118 </span><span class="lineCov"> 539878 : u = v[nd + 5];</span></a>
|
|
<a name="2119"><span class="lineNum"> 2119 </span> : </a>
|
|
<a name="2120"><span class="lineNum"> 2120 </span> : /* special case with only one data disk */</a>
|
|
<a name="2121"><span class="lineNum"> 2121 </span><span class="lineCov"> 539878 : if (l == 0) {</span></a>
|
|
<a name="2122"><span class="lineNum"> 2122 </span><span class="lineCov"> 7 : for (i = 0; i < 6; ++i)</span></a>
|
|
<a name="2123"><span class="lineNum"> 2123 </span><span class="lineCov"> 6 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="2124"><span class="lineNum"> 2124 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="2125"><span class="lineNum"> 2125 </span> : }</a>
|
|
<a name="2126"><span class="lineNum"> 2126 </span> : </a>
|
|
<a name="2127"><span class="lineNum"> 2127 </span> : raid_sse_begin();</a>
|
|
<a name="2128"><span class="lineNum"> 2128 </span> : </a>
|
|
<a name="2129"><span class="lineNum"> 2129 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="2130"><span class="lineNum"> 2130 </span><span class="lineCov"> 539877 : asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="2131"><span class="lineNum"> 2131 </span><span class="lineCov"> 539877 : asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2132"><span class="lineNum"> 2132 </span> : </a>
|
|
<a name="2133"><span class="lineNum"> 2133 </span><span class="lineCov"> 35287797 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="2134"><span class="lineNum"> 2134 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="2135"><span class="lineNum"> 2135 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="2136"><span class="lineNum"> 2136 </span> : </a>
|
|
<a name="2137"><span class="lineNum"> 2137 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %xmm10,%xmm0");</span></a>
|
|
<a name="2138"><span class="lineNum"> 2138 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %xmm10,%xmm1");</span></a>
|
|
<a name="2139"><span class="lineNum"> 2139 </span> : </a>
|
|
<a name="2140"><span class="lineNum"> 2140 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %xmm10,%xmm11");</span></a>
|
|
<a name="2141"><span class="lineNum"> 2141 </span><span class="lineCov"> 34747920 : asm volatile ("psrlw $4,%xmm11");</span></a>
|
|
<a name="2142"><span class="lineNum"> 2142 </span><span class="lineCov"> 34747920 : asm volatile ("pand %xmm15,%xmm10");</span></a>
|
|
<a name="2143"><span class="lineNum"> 2143 </span><span class="lineCov"> 34747920 : asm volatile ("pand %xmm15,%xmm11");</span></a>
|
|
<a name="2144"><span class="lineNum"> 2144 </span> : </a>
|
|
<a name="2145"><span class="lineNum"> 2145 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="2146"><span class="lineNum"> 2146 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="2147"><span class="lineNum"> 2147 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm10,%xmm2");</span></a>
|
|
<a name="2148"><span class="lineNum"> 2148 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2149"><span class="lineNum"> 2149 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm13,%xmm2");</span></a>
|
|
<a name="2150"><span class="lineNum"> 2150 </span> : </a>
|
|
<a name="2151"><span class="lineNum"> 2151 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="2152"><span class="lineNum"> 2152 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="2153"><span class="lineNum"> 2153 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm10,%xmm3");</span></a>
|
|
<a name="2154"><span class="lineNum"> 2154 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2155"><span class="lineNum"> 2155 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm13,%xmm3");</span></a>
|
|
<a name="2156"><span class="lineNum"> 2156 </span> : </a>
|
|
<a name="2157"><span class="lineNum"> 2157 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm4" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="2158"><span class="lineNum"> 2158 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="2159"><span class="lineNum"> 2159 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm10,%xmm4");</span></a>
|
|
<a name="2160"><span class="lineNum"> 2160 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2161"><span class="lineNum"> 2161 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm13,%xmm4");</span></a>
|
|
<a name="2162"><span class="lineNum"> 2162 </span> : </a>
|
|
<a name="2163"><span class="lineNum"> 2163 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm5" : : "m" (gfgenpshufb[l][3][0][0]));</span></a>
|
|
<a name="2164"><span class="lineNum"> 2164 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[l][3][1][0]));</span></a>
|
|
<a name="2165"><span class="lineNum"> 2165 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm10,%xmm5");</span></a>
|
|
<a name="2166"><span class="lineNum"> 2166 </span><span class="lineCov"> 34747920 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2167"><span class="lineNum"> 2167 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm13,%xmm5");</span></a>
|
|
<a name="2168"><span class="lineNum"> 2168 </span> : </a>
|
|
<a name="2169"><span class="lineNum"> 2169 </span> : /* intermediate disks */</a>
|
|
<a name="2170"><span class="lineNum"> 2170 </span><span class="lineCov"> 174133232 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="2171"><span class="lineNum"> 2171 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="2172"><span class="lineNum"> 2172 </span> : </a>
|
|
<a name="2173"><span class="lineNum"> 2173 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm11,%xmm11");</span></a>
|
|
<a name="2174"><span class="lineNum"> 2174 </span><span class="lineCov"> 139385312 : asm volatile ("pcmpgtb %xmm1,%xmm11");</span></a>
|
|
<a name="2175"><span class="lineNum"> 2175 </span><span class="lineCov"> 139385312 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="2176"><span class="lineNum"> 2176 </span><span class="lineCov"> 139385312 : asm volatile ("pand %xmm14,%xmm11");</span></a>
|
|
<a name="2177"><span class="lineNum"> 2177 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm11,%xmm1");</span></a>
|
|
<a name="2178"><span class="lineNum"> 2178 </span> : </a>
|
|
<a name="2179"><span class="lineNum"> 2179 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm10,%xmm0");</span></a>
|
|
<a name="2180"><span class="lineNum"> 2180 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm10,%xmm1");</span></a>
|
|
<a name="2181"><span class="lineNum"> 2181 </span> : </a>
|
|
<a name="2182"><span class="lineNum"> 2182 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %xmm10,%xmm11");</span></a>
|
|
<a name="2183"><span class="lineNum"> 2183 </span><span class="lineCov"> 139385312 : asm volatile ("psrlw $4,%xmm11");</span></a>
|
|
<a name="2184"><span class="lineNum"> 2184 </span><span class="lineCov"> 139385312 : asm volatile ("pand %xmm15,%xmm10");</span></a>
|
|
<a name="2185"><span class="lineNum"> 2185 </span><span class="lineCov"> 139385312 : asm volatile ("pand %xmm15,%xmm11");</span></a>
|
|
<a name="2186"><span class="lineNum"> 2186 </span> : </a>
|
|
<a name="2187"><span class="lineNum"> 2187 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="2188"><span class="lineNum"> 2188 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="2189"><span class="lineNum"> 2189 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="2190"><span class="lineNum"> 2190 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2191"><span class="lineNum"> 2191 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm12,%xmm2");</span></a>
|
|
<a name="2192"><span class="lineNum"> 2192 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm13,%xmm2");</span></a>
|
|
<a name="2193"><span class="lineNum"> 2193 </span> : </a>
|
|
<a name="2194"><span class="lineNum"> 2194 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="2195"><span class="lineNum"> 2195 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="2196"><span class="lineNum"> 2196 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="2197"><span class="lineNum"> 2197 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2198"><span class="lineNum"> 2198 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm12,%xmm3");</span></a>
|
|
<a name="2199"><span class="lineNum"> 2199 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm13,%xmm3");</span></a>
|
|
<a name="2200"><span class="lineNum"> 2200 </span> : </a>
|
|
<a name="2201"><span class="lineNum"> 2201 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="2202"><span class="lineNum"> 2202 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="2203"><span class="lineNum"> 2203 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="2204"><span class="lineNum"> 2204 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2205"><span class="lineNum"> 2205 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm12,%xmm4");</span></a>
|
|
<a name="2206"><span class="lineNum"> 2206 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm13,%xmm4");</span></a>
|
|
<a name="2207"><span class="lineNum"> 2207 </span> : </a>
|
|
<a name="2208"><span class="lineNum"> 2208 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0]));</span></a>
|
|
<a name="2209"><span class="lineNum"> 2209 </span><span class="lineCov"> 139385312 : asm volatile ("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0]));</span></a>
|
|
<a name="2210"><span class="lineNum"> 2210 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm10,%xmm12");</span></a>
|
|
<a name="2211"><span class="lineNum"> 2211 </span><span class="lineCov"> 139385312 : asm volatile ("pshufb %xmm11,%xmm13");</span></a>
|
|
<a name="2212"><span class="lineNum"> 2212 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm12,%xmm5");</span></a>
|
|
<a name="2213"><span class="lineNum"> 2213 </span><span class="lineCov"> 139385312 : asm volatile ("pxor %xmm13,%xmm5");</span></a>
|
|
<a name="2214"><span class="lineNum"> 2214 </span> : }</a>
|
|
<a name="2215"><span class="lineNum"> 2215 </span> : </a>
|
|
<a name="2216"><span class="lineNum"> 2216 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="2217"><span class="lineNum"> 2217 </span><span class="lineCov"> 34747920 : asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="2218"><span class="lineNum"> 2218 </span> : </a>
|
|
<a name="2219"><span class="lineNum"> 2219 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm11,%xmm11");</span></a>
|
|
<a name="2220"><span class="lineNum"> 2220 </span><span class="lineCov"> 34747920 : asm volatile ("pcmpgtb %xmm1,%xmm11");</span></a>
|
|
<a name="2221"><span class="lineNum"> 2221 </span><span class="lineCov"> 34747920 : asm volatile ("paddb %xmm1,%xmm1");</span></a>
|
|
<a name="2222"><span class="lineNum"> 2222 </span><span class="lineCov"> 34747920 : asm volatile ("pand %xmm14,%xmm11");</span></a>
|
|
<a name="2223"><span class="lineNum"> 2223 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm11,%xmm1");</span></a>
|
|
<a name="2224"><span class="lineNum"> 2224 </span> : </a>
|
|
<a name="2225"><span class="lineNum"> 2225 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm10,%xmm0");</span></a>
|
|
<a name="2226"><span class="lineNum"> 2226 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm10,%xmm1");</span></a>
|
|
<a name="2227"><span class="lineNum"> 2227 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm10,%xmm2");</span></a>
|
|
<a name="2228"><span class="lineNum"> 2228 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm10,%xmm3");</span></a>
|
|
<a name="2229"><span class="lineNum"> 2229 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm10,%xmm4");</span></a>
|
|
<a name="2230"><span class="lineNum"> 2230 </span><span class="lineCov"> 34747920 : asm volatile ("pxor %xmm10,%xmm5");</span></a>
|
|
<a name="2231"><span class="lineNum"> 2231 </span> : </a>
|
|
<a name="2232"><span class="lineNum"> 2232 </span><span class="lineCov"> 34747920 : asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="2233"><span class="lineNum"> 2233 </span><span class="lineCov"> 34747920 : asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="2234"><span class="lineNum"> 2234 </span><span class="lineCov"> 34747920 : asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="2235"><span class="lineNum"> 2235 </span><span class="lineCov"> 34747920 : asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="2236"><span class="lineNum"> 2236 </span><span class="lineCov"> 34747920 : asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i]));</span></a>
|
|
<a name="2237"><span class="lineNum"> 2237 </span><span class="lineCov"> 34747920 : asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i]));</span></a>
|
|
<a name="2238"><span class="lineNum"> 2238 </span> : }</a>
|
|
<a name="2239"><span class="lineNum"> 2239 </span> : </a>
|
|
<a name="2240"><span class="lineNum"> 2240 </span> : raid_sse_end();</a>
|
|
<a name="2241"><span class="lineNum"> 2241 </span> : }</a>
|
|
<a name="2242"><span class="lineNum"> 2242 </span> : #endif</a>
|
|
<a name="2243"><span class="lineNum"> 2243 </span> : </a>
|
|
<a name="2244"><span class="lineNum"> 2244 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX2)</a>
|
|
<a name="2245"><span class="lineNum"> 2245 </span> : /*</a>
|
|
<a name="2246"><span class="lineNum"> 2246 </span> : * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation</a>
|
|
<a name="2247"><span class="lineNum"> 2247 </span> : *</a>
|
|
<a name="2248"><span class="lineNum"> 2248 </span> : * Note that it uses 16 registers, meaning that x64 is required.</a>
|
|
<a name="2249"><span class="lineNum"> 2249 </span> : */</a>
|
|
<a name="2250"><span class="lineNum"> 2250 </span><span class="lineCov"> 6 : void raid_gen6_avx2ext(int nd, size_t size, void **vv)</span></a>
|
|
<a name="2251"><span class="lineNum"> 2251 </span> : {</a>
|
|
<a name="2252"><span class="lineNum"> 2252 </span><span class="lineCov"> 6 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2253"><span class="lineNum"> 2253 </span> : uint8_t *p;</a>
|
|
<a name="2254"><span class="lineNum"> 2254 </span> : uint8_t *q;</a>
|
|
<a name="2255"><span class="lineNum"> 2255 </span> : uint8_t *r;</a>
|
|
<a name="2256"><span class="lineNum"> 2256 </span> : uint8_t *s;</a>
|
|
<a name="2257"><span class="lineNum"> 2257 </span> : uint8_t *t;</a>
|
|
<a name="2258"><span class="lineNum"> 2258 </span> : uint8_t *u;</a>
|
|
<a name="2259"><span class="lineNum"> 2259 </span> : int d, l;</a>
|
|
<a name="2260"><span class="lineNum"> 2260 </span> : size_t i;</a>
|
|
<a name="2261"><span class="lineNum"> 2261 </span> : </a>
|
|
<a name="2262"><span class="lineNum"> 2262 </span><span class="lineCov"> 6 : l = nd - 1;</span></a>
|
|
<a name="2263"><span class="lineNum"> 2263 </span><span class="lineCov"> 6 : p = v[nd];</span></a>
|
|
<a name="2264"><span class="lineNum"> 2264 </span><span class="lineCov"> 6 : q = v[nd + 1];</span></a>
|
|
<a name="2265"><span class="lineNum"> 2265 </span><span class="lineCov"> 6 : r = v[nd + 2];</span></a>
|
|
<a name="2266"><span class="lineNum"> 2266 </span><span class="lineCov"> 6 : s = v[nd + 3];</span></a>
|
|
<a name="2267"><span class="lineNum"> 2267 </span><span class="lineCov"> 6 : t = v[nd + 4];</span></a>
|
|
<a name="2268"><span class="lineNum"> 2268 </span><span class="lineCov"> 6 : u = v[nd + 5];</span></a>
|
|
<a name="2269"><span class="lineNum"> 2269 </span> : </a>
|
|
<a name="2270"><span class="lineNum"> 2270 </span> : /* special case with only one data disk */</a>
|
|
<a name="2271"><span class="lineNum"> 2271 </span><span class="lineCov"> 6 : if (l == 0) {</span></a>
|
|
<a name="2272"><span class="lineNum"> 2272 </span><span class="lineCov"> 7 : for (i = 0; i < 6; ++i)</span></a>
|
|
<a name="2273"><span class="lineNum"> 2273 </span><span class="lineCov"> 6 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="2274"><span class="lineNum"> 2274 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="2275"><span class="lineNum"> 2275 </span> : }</a>
|
|
<a name="2276"><span class="lineNum"> 2276 </span> : </a>
|
|
<a name="2277"><span class="lineNum"> 2277 </span> : raid_avx_begin();</a>
|
|
<a name="2278"><span class="lineNum"> 2278 </span> : </a>
|
|
<a name="2279"><span class="lineNum"> 2279 </span> : /*</a>
|
|
<a name="2280"><span class="lineNum"> 2280 </span> : * Register mapping:</a>
|
|
<a name="2281"><span class="lineNum"> 2281 </span> : * </a>
|
|
<a name="2282"><span class="lineNum"> 2282 </span> : * 0,1,2,3,4,5 PARITY</a>
|
|
<a name="2283"><span class="lineNum"> 2283 </span> : * 6 ZERO</a>
|
|
<a name="2284"><span class="lineNum"> 2284 </span> : * 10,11 MUL BY 2</a>
|
|
<a name="2285"><span class="lineNum"> 2285 </span> : * 12,13 PSHUFB</a>
|
|
<a name="2286"><span class="lineNum"> 2286 </span> : * 14 POLY</a>
|
|
<a name="2287"><span class="lineNum"> 2287 </span> : * 15 LOW4</a>
|
|
<a name="2288"><span class="lineNum"> 2288 </span> : */</a>
|
|
<a name="2289"><span class="lineNum"> 2289 </span> : </a>
|
|
<a name="2290"><span class="lineNum"> 2290 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="2291"><span class="lineNum"> 2291 </span><span class="lineCov"> 5 : asm volatile ("vpxor %ymm6,%ymm6,%ymm6");</span></a>
|
|
<a name="2292"><span class="lineNum"> 2292 </span><span class="lineCov"> 5 : asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="2293"><span class="lineNum"> 2293 </span><span class="lineCov"> 5 : asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2294"><span class="lineNum"> 2294 </span> : </a>
|
|
<a name="2295"><span class="lineNum"> 2295 </span><span class="lineCov"> 32781 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="2296"><span class="lineNum"> 2296 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="2297"><span class="lineNum"> 2297 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="2298"><span class="lineNum"> 2298 </span> : </a>
|
|
<a name="2299"><span class="lineNum"> 2299 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %ymm10,%ymm0");</span></a>
|
|
<a name="2300"><span class="lineNum"> 2300 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %ymm10,%ymm1");</span></a>
|
|
<a name="2301"><span class="lineNum"> 2301 </span> : </a>
|
|
<a name="2302"><span class="lineNum"> 2302 </span><span class="lineCov"> 32776 : asm volatile ("vpsrlw $4,%ymm10,%ymm11");</span></a>
|
|
<a name="2303"><span class="lineNum"> 2303 </span><span class="lineCov"> 32776 : asm volatile ("vpand %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="2304"><span class="lineNum"> 2304 </span><span class="lineCov"> 32776 : asm volatile ("vpand %ymm15,%ymm11,%ymm11");</span></a>
|
|
<a name="2305"><span class="lineNum"> 2305 </span> : </a>
|
|
<a name="2306"><span class="lineNum"> 2306 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="2307"><span class="lineNum"> 2307 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="2308"><span class="lineNum"> 2308 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm2,%ymm2");</span></a>
|
|
<a name="2309"><span class="lineNum"> 2309 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2310"><span class="lineNum"> 2310 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm2,%ymm2");</span></a>
|
|
<a name="2311"><span class="lineNum"> 2311 </span> : </a>
|
|
<a name="2312"><span class="lineNum"> 2312 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="2313"><span class="lineNum"> 2313 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="2314"><span class="lineNum"> 2314 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm3,%ymm3");</span></a>
|
|
<a name="2315"><span class="lineNum"> 2315 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2316"><span class="lineNum"> 2316 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm3,%ymm3");</span></a>
|
|
<a name="2317"><span class="lineNum"> 2317 </span> : </a>
|
|
<a name="2318"><span class="lineNum"> 2318 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="2319"><span class="lineNum"> 2319 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="2320"><span class="lineNum"> 2320 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm4,%ymm4");</span></a>
|
|
<a name="2321"><span class="lineNum"> 2321 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2322"><span class="lineNum"> 2322 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm4,%ymm4");</span></a>
|
|
<a name="2323"><span class="lineNum"> 2323 </span> : </a>
|
|
<a name="2324"><span class="lineNum"> 2324 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfgenpshufb[l][3][0][0]));</span></a>
|
|
<a name="2325"><span class="lineNum"> 2325 </span><span class="lineCov"> 32776 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[l][3][1][0]));</span></a>
|
|
<a name="2326"><span class="lineNum"> 2326 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm10,%ymm5,%ymm5");</span></a>
|
|
<a name="2327"><span class="lineNum"> 2327 </span><span class="lineCov"> 32776 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2328"><span class="lineNum"> 2328 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm13,%ymm5,%ymm5");</span></a>
|
|
<a name="2329"><span class="lineNum"> 2329 </span> : </a>
|
|
<a name="2330"><span class="lineNum"> 2330 </span> : /* intermediate disks */</a>
|
|
<a name="2331"><span class="lineNum"> 2331 </span><span class="lineCov"> 229624 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="2332"><span class="lineNum"> 2332 </span><span class="lineCov"> 196848 : asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="2333"><span class="lineNum"> 2333 </span> : </a>
|
|
<a name="2334"><span class="lineNum"> 2334 </span><span class="lineCov"> 196848 : asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");</span></a>
|
|
<a name="2335"><span class="lineNum"> 2335 </span><span class="lineCov"> 196848 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="2336"><span class="lineNum"> 2336 </span><span class="lineCov"> 196848 : asm volatile ("vpand %ymm14,%ymm11,%ymm11");</span></a>
|
|
<a name="2337"><span class="lineNum"> 2337 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm11,%ymm1,%ymm1");</span></a>
|
|
<a name="2338"><span class="lineNum"> 2338 </span> : </a>
|
|
<a name="2339"><span class="lineNum"> 2339 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm10,%ymm0,%ymm0");</span></a>
|
|
<a name="2340"><span class="lineNum"> 2340 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm10,%ymm1,%ymm1");</span></a>
|
|
<a name="2341"><span class="lineNum"> 2341 </span> : </a>
|
|
<a name="2342"><span class="lineNum"> 2342 </span><span class="lineCov"> 196848 : asm volatile ("vpsrlw $4,%ymm10,%ymm11");</span></a>
|
|
<a name="2343"><span class="lineNum"> 2343 </span><span class="lineCov"> 196848 : asm volatile ("vpand %ymm15,%ymm10,%ymm10");</span></a>
|
|
<a name="2344"><span class="lineNum"> 2344 </span><span class="lineCov"> 196848 : asm volatile ("vpand %ymm15,%ymm11,%ymm11");</span></a>
|
|
<a name="2345"><span class="lineNum"> 2345 </span> : </a>
|
|
<a name="2346"><span class="lineNum"> 2346 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="2347"><span class="lineNum"> 2347 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="2348"><span class="lineNum"> 2348 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="2349"><span class="lineNum"> 2349 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2350"><span class="lineNum"> 2350 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm2,%ymm2");</span></a>
|
|
<a name="2351"><span class="lineNum"> 2351 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm2,%ymm2");</span></a>
|
|
<a name="2352"><span class="lineNum"> 2352 </span> : </a>
|
|
<a name="2353"><span class="lineNum"> 2353 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="2354"><span class="lineNum"> 2354 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="2355"><span class="lineNum"> 2355 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="2356"><span class="lineNum"> 2356 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2357"><span class="lineNum"> 2357 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm3,%ymm3");</span></a>
|
|
<a name="2358"><span class="lineNum"> 2358 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm3,%ymm3");</span></a>
|
|
<a name="2359"><span class="lineNum"> 2359 </span> : </a>
|
|
<a name="2360"><span class="lineNum"> 2360 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="2361"><span class="lineNum"> 2361 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="2362"><span class="lineNum"> 2362 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="2363"><span class="lineNum"> 2363 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2364"><span class="lineNum"> 2364 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm4,%ymm4");</span></a>
|
|
<a name="2365"><span class="lineNum"> 2365 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm4,%ymm4");</span></a>
|
|
<a name="2366"><span class="lineNum"> 2366 </span> : </a>
|
|
<a name="2367"><span class="lineNum"> 2367 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (gfgenpshufb[d][3][0][0]));</span></a>
|
|
<a name="2368"><span class="lineNum"> 2368 </span><span class="lineCov"> 196848 : asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (gfgenpshufb[d][3][1][0]));</span></a>
|
|
<a name="2369"><span class="lineNum"> 2369 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");</span></a>
|
|
<a name="2370"><span class="lineNum"> 2370 </span><span class="lineCov"> 196848 : asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");</span></a>
|
|
<a name="2371"><span class="lineNum"> 2371 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm12,%ymm5,%ymm5");</span></a>
|
|
<a name="2372"><span class="lineNum"> 2372 </span><span class="lineCov"> 196848 : asm volatile ("vpxor %ymm13,%ymm5,%ymm5");</span></a>
|
|
<a name="2373"><span class="lineNum"> 2373 </span> : }</a>
|
|
<a name="2374"><span class="lineNum"> 2374 </span> : </a>
|
|
<a name="2375"><span class="lineNum"> 2375 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="2376"><span class="lineNum"> 2376 </span><span class="lineCov"> 32776 : asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="2377"><span class="lineNum"> 2377 </span> : </a>
|
|
<a name="2378"><span class="lineNum"> 2378 </span><span class="lineCov"> 32776 : asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");</span></a>
|
|
<a name="2379"><span class="lineNum"> 2379 </span><span class="lineCov"> 32776 : asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="2380"><span class="lineNum"> 2380 </span><span class="lineCov"> 32776 : asm volatile ("vpand %ymm14,%ymm11,%ymm11");</span></a>
|
|
<a name="2381"><span class="lineNum"> 2381 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm11,%ymm1,%ymm1");</span></a>
|
|
<a name="2382"><span class="lineNum"> 2382 </span> : </a>
|
|
<a name="2383"><span class="lineNum"> 2383 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm0,%ymm0");</span></a>
|
|
<a name="2384"><span class="lineNum"> 2384 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm1,%ymm1");</span></a>
|
|
<a name="2385"><span class="lineNum"> 2385 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm2,%ymm2");</span></a>
|
|
<a name="2386"><span class="lineNum"> 2386 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm3,%ymm3");</span></a>
|
|
<a name="2387"><span class="lineNum"> 2387 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm4,%ymm4");</span></a>
|
|
<a name="2388"><span class="lineNum"> 2388 </span><span class="lineCov"> 32776 : asm volatile ("vpxor %ymm10,%ymm5,%ymm5");</span></a>
|
|
<a name="2389"><span class="lineNum"> 2389 </span> : </a>
|
|
<a name="2390"><span class="lineNum"> 2390 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="2391"><span class="lineNum"> 2391 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="2392"><span class="lineNum"> 2392 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="2393"><span class="lineNum"> 2393 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="2394"><span class="lineNum"> 2394 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i]));</span></a>
|
|
<a name="2395"><span class="lineNum"> 2395 </span><span class="lineCov"> 32776 : asm volatile ("vmovntdq %%ymm5,%0" : "=m" (u[i]));</span></a>
|
|
<a name="2396"><span class="lineNum"> 2396 </span> : }</a>
|
|
<a name="2397"><span class="lineNum"> 2397 </span> : </a>
|
|
<a name="2398"><span class="lineNum"> 2398 </span> : raid_avx_end();</a>
|
|
<a name="2399"><span class="lineNum"> 2399 </span> : }</a>
|
|
<a name="2400"><span class="lineNum"> 2400 </span> : #endif</a>
|
|
<a name="2401"><span class="lineNum"> 2401 </span> : </a>
|
|
<a name="2402"><span class="lineNum"> 2402 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="2403"><span class="lineNum"> 2403 </span> : /*</a>
|
|
<a name="2404"><span class="lineNum"> 2404 </span> : * GEN6 (hexa parity with Cauchy matrix) AVX512BW implementation</a>
|
|
<a name="2405"><span class="lineNum"> 2405 </span> : */</a>
|
|
<a name="2406"><span class="lineNum"> 2406 </span><span class="lineCov"> 49514 : void raid_gen6_avx512bw(int nd, size_t size, void **vv)</span></a>
|
|
<a name="2407"><span class="lineNum"> 2407 </span> : {</a>
|
|
<a name="2408"><span class="lineNum"> 2408 </span><span class="lineCov"> 49514 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2409"><span class="lineNum"> 2409 </span> : uint8_t *p;</a>
|
|
<a name="2410"><span class="lineNum"> 2410 </span> : uint8_t *q;</a>
|
|
<a name="2411"><span class="lineNum"> 2411 </span> : uint8_t *r;</a>
|
|
<a name="2412"><span class="lineNum"> 2412 </span> : uint8_t *s;</a>
|
|
<a name="2413"><span class="lineNum"> 2413 </span> : uint8_t *t;</a>
|
|
<a name="2414"><span class="lineNum"> 2414 </span> : uint8_t *u;</a>
|
|
<a name="2415"><span class="lineNum"> 2415 </span> : int d, l;</a>
|
|
<a name="2416"><span class="lineNum"> 2416 </span> : size_t i;</a>
|
|
<a name="2417"><span class="lineNum"> 2417 </span> : </a>
|
|
<a name="2418"><span class="lineNum"> 2418 </span><span class="lineCov"> 49514 : l = nd - 1;</span></a>
|
|
<a name="2419"><span class="lineNum"> 2419 </span><span class="lineCov"> 49514 : p = v[nd];</span></a>
|
|
<a name="2420"><span class="lineNum"> 2420 </span><span class="lineCov"> 49514 : q = v[nd + 1];</span></a>
|
|
<a name="2421"><span class="lineNum"> 2421 </span><span class="lineCov"> 49514 : r = v[nd + 2];</span></a>
|
|
<a name="2422"><span class="lineNum"> 2422 </span><span class="lineCov"> 49514 : s = v[nd + 3];</span></a>
|
|
<a name="2423"><span class="lineNum"> 2423 </span><span class="lineCov"> 49514 : t = v[nd + 4];</span></a>
|
|
<a name="2424"><span class="lineNum"> 2424 </span><span class="lineCov"> 49514 : u = v[nd + 5];</span></a>
|
|
<a name="2425"><span class="lineNum"> 2425 </span> : </a>
|
|
<a name="2426"><span class="lineNum"> 2426 </span> : /* special case with only one data disk */</a>
|
|
<a name="2427"><span class="lineNum"> 2427 </span><span class="lineCov"> 49514 : if (l == 0) {</span></a>
|
|
<a name="2428"><span class="lineNum"> 2428 </span><span class="lineCov"> 7 : for (i = 0; i < 6; ++i)</span></a>
|
|
<a name="2429"><span class="lineNum"> 2429 </span><span class="lineCov"> 6 : memcpy(v[1 + i], v[0], size);</span></a>
|
|
<a name="2430"><span class="lineNum"> 2430 </span><span class="lineCov"> 1 : return;</span></a>
|
|
<a name="2431"><span class="lineNum"> 2431 </span> : }</a>
|
|
<a name="2432"><span class="lineNum"> 2432 </span> : </a>
|
|
<a name="2433"><span class="lineNum"> 2433 </span> : raid_avx_begin();</a>
|
|
<a name="2434"><span class="lineNum"> 2434 </span> : </a>
|
|
<a name="2435"><span class="lineNum"> 2435 </span> : /*</a>
|
|
<a name="2436"><span class="lineNum"> 2436 </span> : * Register mapping:</a>
|
|
<a name="2437"><span class="lineNum"> 2437 </span> : * </a>
|
|
<a name="2438"><span class="lineNum"> 2438 </span> : * 0,1,2,3,4,5 PARITY</a>
|
|
<a name="2439"><span class="lineNum"> 2439 </span> : * 6 ZERO</a>
|
|
<a name="2440"><span class="lineNum"> 2440 </span> : * 10,11 MUL BY 2</a>
|
|
<a name="2441"><span class="lineNum"> 2441 </span> : * 12,13 PSHUFB</a>
|
|
<a name="2442"><span class="lineNum"> 2442 </span> : * 14 POLY</a>
|
|
<a name="2443"><span class="lineNum"> 2443 </span> : * 15 LOW4</a>
|
|
<a name="2444"><span class="lineNum"> 2444 </span> : */</a>
|
|
<a name="2445"><span class="lineNum"> 2445 </span> : </a>
|
|
<a name="2446"><span class="lineNum"> 2446 </span> : /* generic case with at least two data disks */</a>
|
|
<a name="2447"><span class="lineNum"> 2447 </span><span class="lineCov"> 49513 : asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="2448"><span class="lineNum"> 2448 </span><span class="lineCov"> 49513 : asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));</span></a>
|
|
<a name="2449"><span class="lineNum"> 2449 </span><span class="lineCov"> 49513 : asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2450"><span class="lineNum"> 2450 </span> : </a>
|
|
<a name="2451"><span class="lineNum"> 2451 </span><span class="lineCov"> 256109 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="2452"><span class="lineNum"> 2452 </span> : /* last disk without the by two multiplication */</a>
|
|
<a name="2453"><span class="lineNum"> 2453 </span><span class="lineCov"> 206596 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[l][i]));</span></a>
|
|
<a name="2454"><span class="lineNum"> 2454 </span> : </a>
|
|
<a name="2455"><span class="lineNum"> 2455 </span><span class="lineCov"> 206596 : asm volatile ("vmovdqa64 %zmm10,%zmm0");</span></a>
|
|
<a name="2456"><span class="lineNum"> 2456 </span><span class="lineCov"> 206596 : asm volatile ("vmovdqa64 %zmm10,%zmm1");</span></a>
|
|
<a name="2457"><span class="lineNum"> 2457 </span> : </a>
|
|
<a name="2458"><span class="lineNum"> 2458 </span><span class="lineCov"> 206596 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="2459"><span class="lineNum"> 2459 </span><span class="lineCov"> 206596 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="2460"><span class="lineNum"> 2460 </span><span class="lineCov"> 206596 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="2461"><span class="lineNum"> 2461 </span> : </a>
|
|
<a name="2462"><span class="lineNum"> 2462 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfgenpshufb[l][0][0][0]));</span></a>
|
|
<a name="2463"><span class="lineNum"> 2463 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][0][1][0]));</span></a>
|
|
<a name="2464"><span class="lineNum"> 2464 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="2465"><span class="lineNum"> 2465 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2466"><span class="lineNum"> 2466 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="2467"><span class="lineNum"> 2467 </span> : </a>
|
|
<a name="2468"><span class="lineNum"> 2468 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfgenpshufb[l][1][0][0]));</span></a>
|
|
<a name="2469"><span class="lineNum"> 2469 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][1][1][0]));</span></a>
|
|
<a name="2470"><span class="lineNum"> 2470 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm10,%zmm3,%zmm3");</span></a>
|
|
<a name="2471"><span class="lineNum"> 2471 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2472"><span class="lineNum"> 2472 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");</span></a>
|
|
<a name="2473"><span class="lineNum"> 2473 </span> : </a>
|
|
<a name="2474"><span class="lineNum"> 2474 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfgenpshufb[l][2][0][0]));</span></a>
|
|
<a name="2475"><span class="lineNum"> 2475 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][2][1][0]));</span></a>
|
|
<a name="2476"><span class="lineNum"> 2476 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm10,%zmm4,%zmm4");</span></a>
|
|
<a name="2477"><span class="lineNum"> 2477 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2478"><span class="lineNum"> 2478 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");</span></a>
|
|
<a name="2479"><span class="lineNum"> 2479 </span> : </a>
|
|
<a name="2480"><span class="lineNum"> 2480 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfgenpshufb[l][3][0][0]));</span></a>
|
|
<a name="2481"><span class="lineNum"> 2481 </span><span class="lineCov"> 206596 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[l][3][1][0]));</span></a>
|
|
<a name="2482"><span class="lineNum"> 2482 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm10,%zmm5,%zmm5");</span></a>
|
|
<a name="2483"><span class="lineNum"> 2483 </span><span class="lineCov"> 206596 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2484"><span class="lineNum"> 2484 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm13,%zmm5,%zmm5");</span></a>
|
|
<a name="2485"><span class="lineNum"> 2485 </span> : </a>
|
|
<a name="2486"><span class="lineNum"> 2486 </span> : /* intermediate disks */</a>
|
|
<a name="2487"><span class="lineNum"> 2487 </span><span class="lineCov"> 2241404 : for (d = l - 1; d > 0; --d) {</span></a>
|
|
<a name="2488"><span class="lineNum"> 2488 </span><span class="lineCov"> 2034808 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[d][i]));</span></a>
|
|
<a name="2489"><span class="lineNum"> 2489 </span> : </a>
|
|
<a name="2490"><span class="lineNum"> 2490 </span><span class="lineCov"> 2034808 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="2491"><span class="lineNum"> 2491 </span><span class="lineCov"> 2034808 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="2492"><span class="lineNum"> 2492 </span><span class="lineCov"> 2034808 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="2493"><span class="lineNum"> 2493 </span><span class="lineCov"> 2034808 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="2494"><span class="lineNum"> 2494 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="2495"><span class="lineNum"> 2495 </span> : </a>
|
|
<a name="2496"><span class="lineNum"> 2496 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="2497"><span class="lineNum"> 2497 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="2498"><span class="lineNum"> 2498 </span> : </a>
|
|
<a name="2499"><span class="lineNum"> 2499 </span><span class="lineCov"> 2034808 : asm volatile ("vpsrlw $4,%zmm10,%zmm11");</span></a>
|
|
<a name="2500"><span class="lineNum"> 2500 </span><span class="lineCov"> 2034808 : asm volatile ("vpandq %zmm15,%zmm10,%zmm10");</span></a>
|
|
<a name="2501"><span class="lineNum"> 2501 </span><span class="lineCov"> 2034808 : asm volatile ("vpandq %zmm15,%zmm11,%zmm11");</span></a>
|
|
<a name="2502"><span class="lineNum"> 2502 </span> : </a>
|
|
<a name="2503"><span class="lineNum"> 2503 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][0][0][0]));</span></a>
|
|
<a name="2504"><span class="lineNum"> 2504 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][0][1][0]));</span></a>
|
|
<a name="2505"><span class="lineNum"> 2505 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="2506"><span class="lineNum"> 2506 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2507"><span class="lineNum"> 2507 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");</span></a>
|
|
<a name="2508"><span class="lineNum"> 2508 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");</span></a>
|
|
<a name="2509"><span class="lineNum"> 2509 </span> : </a>
|
|
<a name="2510"><span class="lineNum"> 2510 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][1][0][0]));</span></a>
|
|
<a name="2511"><span class="lineNum"> 2511 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][1][1][0]));</span></a>
|
|
<a name="2512"><span class="lineNum"> 2512 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="2513"><span class="lineNum"> 2513 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2514"><span class="lineNum"> 2514 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm12,%zmm3,%zmm3");</span></a>
|
|
<a name="2515"><span class="lineNum"> 2515 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");</span></a>
|
|
<a name="2516"><span class="lineNum"> 2516 </span> : </a>
|
|
<a name="2517"><span class="lineNum"> 2517 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][2][0][0]));</span></a>
|
|
<a name="2518"><span class="lineNum"> 2518 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][2][1][0]));</span></a>
|
|
<a name="2519"><span class="lineNum"> 2519 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="2520"><span class="lineNum"> 2520 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2521"><span class="lineNum"> 2521 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm12,%zmm4,%zmm4");</span></a>
|
|
<a name="2522"><span class="lineNum"> 2522 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm13,%zmm4,%zmm4");</span></a>
|
|
<a name="2523"><span class="lineNum"> 2523 </span> : </a>
|
|
<a name="2524"><span class="lineNum"> 2524 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (gfgenpshufb[d][3][0][0]));</span></a>
|
|
<a name="2525"><span class="lineNum"> 2525 </span><span class="lineCov"> 2034808 : asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (gfgenpshufb[d][3][1][0]));</span></a>
|
|
<a name="2526"><span class="lineNum"> 2526 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm10,%zmm12,%zmm12");</span></a>
|
|
<a name="2527"><span class="lineNum"> 2527 </span><span class="lineCov"> 2034808 : asm volatile ("vpshufb %zmm11,%zmm13,%zmm13");</span></a>
|
|
<a name="2528"><span class="lineNum"> 2528 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm12,%zmm5,%zmm5");</span></a>
|
|
<a name="2529"><span class="lineNum"> 2529 </span><span class="lineCov"> 2034808 : asm volatile ("vpxorq %zmm13,%zmm5,%zmm5");</span></a>
|
|
<a name="2530"><span class="lineNum"> 2530 </span> : }</a>
|
|
<a name="2531"><span class="lineNum"> 2531 </span> : </a>
|
|
<a name="2532"><span class="lineNum"> 2532 </span> : /* first disk with all coefficients at 1 */</a>
|
|
<a name="2533"><span class="lineNum"> 2533 </span><span class="lineCov"> 206596 : asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (v[0][i]));</span></a>
|
|
<a name="2534"><span class="lineNum"> 2534 </span> : </a>
|
|
<a name="2535"><span class="lineNum"> 2535 </span><span class="lineCov"> 206596 : asm volatile ("vpcmpgtb %zmm1,%zmm6,%k1");</span></a>
|
|
<a name="2536"><span class="lineNum"> 2536 </span><span class="lineCov"> 206596 : asm volatile ("vpmovm2b %k1,%zmm11");</span></a>
|
|
<a name="2537"><span class="lineNum"> 2537 </span><span class="lineCov"> 206596 : asm volatile ("vpaddb %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="2538"><span class="lineNum"> 2538 </span><span class="lineCov"> 206596 : asm volatile ("vpandq %zmm14,%zmm11,%zmm11");</span></a>
|
|
<a name="2539"><span class="lineNum"> 2539 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm11,%zmm1,%zmm1");</span></a>
|
|
<a name="2540"><span class="lineNum"> 2540 </span> : </a>
|
|
<a name="2541"><span class="lineNum"> 2541 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm10,%zmm0,%zmm0");</span></a>
|
|
<a name="2542"><span class="lineNum"> 2542 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm10,%zmm1,%zmm1");</span></a>
|
|
<a name="2543"><span class="lineNum"> 2543 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm10,%zmm2,%zmm2");</span></a>
|
|
<a name="2544"><span class="lineNum"> 2544 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm10,%zmm3,%zmm3");</span></a>
|
|
<a name="2545"><span class="lineNum"> 2545 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm10,%zmm4,%zmm4");</span></a>
|
|
<a name="2546"><span class="lineNum"> 2546 </span><span class="lineCov"> 206596 : asm volatile ("vpxorq %zmm10,%zmm5,%zmm5");</span></a>
|
|
<a name="2547"><span class="lineNum"> 2547 </span> : </a>
|
|
<a name="2548"><span class="lineNum"> 2548 </span><span class="lineCov"> 206596 : asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));</span></a>
|
|
<a name="2549"><span class="lineNum"> 2549 </span><span class="lineCov"> 206596 : asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));</span></a>
|
|
<a name="2550"><span class="lineNum"> 2550 </span><span class="lineCov"> 206596 : asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));</span></a>
|
|
<a name="2551"><span class="lineNum"> 2551 </span><span class="lineCov"> 206596 : asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i]));</span></a>
|
|
<a name="2552"><span class="lineNum"> 2552 </span><span class="lineCov"> 206596 : asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i]));</span></a>
|
|
<a name="2553"><span class="lineNum"> 2553 </span><span class="lineCov"> 206596 : asm volatile ("vmovntdq %%zmm5,%0" : "=m" (u[i]));</span></a>
|
|
<a name="2554"><span class="lineNum"> 2554 </span> : }</a>
|
|
<a name="2555"><span class="lineNum"> 2555 </span> : </a>
|
|
<a name="2556"><span class="lineNum"> 2556 </span> : raid_avx_end();</a>
|
|
<a name="2557"><span class="lineNum"> 2557 </span> : }</a>
|
|
<a name="2558"><span class="lineNum"> 2558 </span> : #endif</a>
|
|
<a name="2559"><span class="lineNum"> 2559 </span> : </a>
|
|
<a name="2560"><span class="lineNum"> 2560 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="2561"><span class="lineNum"> 2561 </span> : /*</a>
|
|
<a name="2562"><span class="lineNum"> 2562 </span> : * RAID recovering for one disk SSSE3 implementation</a>
|
|
<a name="2563"><span class="lineNum"> 2563 </span> : */</a>
|
|
<a name="2564"><span class="lineNum"> 2564 </span><span class="lineCov"> 111093 : void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="2565"><span class="lineNum"> 2565 </span> : {</a>
|
|
<a name="2566"><span class="lineNum"> 2566 </span><span class="lineCov"> 111093 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2567"><span class="lineNum"> 2567 </span> : uint8_t *p;</a>
|
|
<a name="2568"><span class="lineNum"> 2568 </span> : uint8_t *pa;</a>
|
|
<a name="2569"><span class="lineNum"> 2569 </span> : uint8_t G;</a>
|
|
<a name="2570"><span class="lineNum"> 2570 </span> : uint8_t V;</a>
|
|
<a name="2571"><span class="lineNum"> 2571 </span> : size_t i;</a>
|
|
<a name="2572"><span class="lineNum"> 2572 </span> : </a>
|
|
<a name="2573"><span class="lineNum"> 2573 </span> : (void)nr; /* unused, it's always 1 */</a>
|
|
<a name="2574"><span class="lineNum"> 2574 </span> : </a>
|
|
<a name="2575"><span class="lineNum"> 2575 </span> : /* if it's RAID5 uses the faster function */</a>
|
|
<a name="2576"><span class="lineNum"> 2576 </span><span class="lineCov"> 111093 : if (ip[0] == 0) {</span></a>
|
|
<a name="2577"><span class="lineNum"> 2577 </span><span class="lineCov"> 100930 : raid_rec1of1(id, nd, size, vv);</span></a>
|
|
<a name="2578"><span class="lineNum"> 2578 </span><span class="lineCov"> 100930 : return;</span></a>
|
|
<a name="2579"><span class="lineNum"> 2579 </span> : }</a>
|
|
<a name="2580"><span class="lineNum"> 2580 </span> : </a>
|
|
<a name="2581"><span class="lineNum"> 2581 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="2582"><span class="lineNum"> 2582 </span><span class="lineCov"> 10163 : G = A(ip[0], id[0]);</span></a>
|
|
<a name="2583"><span class="lineNum"> 2583 </span> : </a>
|
|
<a name="2584"><span class="lineNum"> 2584 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="2585"><span class="lineNum"> 2585 </span><span class="lineCov"> 10163 : V = inv(G);</span></a>
|
|
<a name="2586"><span class="lineNum"> 2586 </span> : </a>
|
|
<a name="2587"><span class="lineNum"> 2587 </span> : /* compute delta parity */</a>
|
|
<a name="2588"><span class="lineNum"> 2588 </span><span class="lineCov"> 10163 : raid_delta_gen(1, id, ip, nd, size, vv);</span></a>
|
|
<a name="2589"><span class="lineNum"> 2589 </span> : </a>
|
|
<a name="2590"><span class="lineNum"> 2590 </span><span class="lineCov"> 10163 : p = v[nd + ip[0]];</span></a>
|
|
<a name="2591"><span class="lineNum"> 2591 </span><span class="lineCov"> 10163 : pa = v[id[0]];</span></a>
|
|
<a name="2592"><span class="lineNum"> 2592 </span> : </a>
|
|
<a name="2593"><span class="lineNum"> 2593 </span> : raid_sse_begin();</a>
|
|
<a name="2594"><span class="lineNum"> 2594 </span> : </a>
|
|
<a name="2595"><span class="lineNum"> 2595 </span><span class="lineCov"> 10163 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2596"><span class="lineNum"> 2596 </span><span class="lineCov"> 10163 : asm volatile ("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0]));</span></a>
|
|
<a name="2597"><span class="lineNum"> 2597 </span><span class="lineCov"> 10163 : asm volatile ("movdqa %0,%%xmm5" : : "m" (gfmulpshufb[V][1][0]));</span></a>
|
|
<a name="2598"><span class="lineNum"> 2598 </span> : </a>
|
|
<a name="2599"><span class="lineNum"> 2599 </span><span class="lineCov"> 934003 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="2600"><span class="lineNum"> 2600 </span><span class="lineCov"> 923840 : asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));</span></a>
|
|
<a name="2601"><span class="lineNum"> 2601 </span><span class="lineCov"> 923840 : asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[i]));</span></a>
|
|
<a name="2602"><span class="lineNum"> 2602 </span><span class="lineCov"> 923840 : asm volatile ("movdqa %xmm4,%xmm2");</span></a>
|
|
<a name="2603"><span class="lineNum"> 2603 </span><span class="lineCov"> 923840 : asm volatile ("movdqa %xmm5,%xmm3");</span></a>
|
|
<a name="2604"><span class="lineNum"> 2604 </span><span class="lineCov"> 923840 : asm volatile ("pxor %xmm0,%xmm1");</span></a>
|
|
<a name="2605"><span class="lineNum"> 2605 </span><span class="lineCov"> 923840 : asm volatile ("movdqa %xmm1,%xmm0");</span></a>
|
|
<a name="2606"><span class="lineNum"> 2606 </span><span class="lineCov"> 923840 : asm volatile ("psrlw $4,%xmm1");</span></a>
|
|
<a name="2607"><span class="lineNum"> 2607 </span><span class="lineCov"> 923840 : asm volatile ("pand %xmm7,%xmm0");</span></a>
|
|
<a name="2608"><span class="lineNum"> 2608 </span><span class="lineCov"> 923840 : asm volatile ("pand %xmm7,%xmm1");</span></a>
|
|
<a name="2609"><span class="lineNum"> 2609 </span><span class="lineCov"> 923840 : asm volatile ("pshufb %xmm0,%xmm2");</span></a>
|
|
<a name="2610"><span class="lineNum"> 2610 </span><span class="lineCov"> 923840 : asm volatile ("pshufb %xmm1,%xmm3");</span></a>
|
|
<a name="2611"><span class="lineNum"> 2611 </span><span class="lineCov"> 923840 : asm volatile ("pxor %xmm3,%xmm2");</span></a>
|
|
<a name="2612"><span class="lineNum"> 2612 </span><span class="lineCov"> 923840 : asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i]));</span></a>
|
|
<a name="2613"><span class="lineNum"> 2613 </span> : }</a>
|
|
<a name="2614"><span class="lineNum"> 2614 </span> : </a>
|
|
<a name="2615"><span class="lineNum"> 2615 </span> : raid_sse_end();</a>
|
|
<a name="2616"><span class="lineNum"> 2616 </span> : }</a>
|
|
<a name="2617"><span class="lineNum"> 2617 </span> : #endif</a>
|
|
<a name="2618"><span class="lineNum"> 2618 </span> : </a>
|
|
<a name="2619"><span class="lineNum"> 2619 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="2620"><span class="lineNum"> 2620 </span> : /*</a>
|
|
<a name="2621"><span class="lineNum"> 2621 </span> : * RAID recovering for two disks SSSE3 implementation</a>
|
|
<a name="2622"><span class="lineNum"> 2622 </span> : */</a>
|
|
<a name="2623"><span class="lineNum"> 2623 </span><span class="lineCov"> 43770 : void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="2624"><span class="lineNum"> 2624 </span><span class="lineCov"> 43770 : {</span></a>
|
|
<a name="2625"><span class="lineNum"> 2625 </span><span class="lineCov"> 43770 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2626"><span class="lineNum"> 2626 </span><span class="lineCov"> 43770 : const int N = 2;</span></a>
|
|
<a name="2627"><span class="lineNum"> 2627 </span><span class="lineCov"> 43770 : uint8_t *p[N];</span></a>
|
|
<a name="2628"><span class="lineNum"> 2628 </span><span class="lineCov"> 43770 : uint8_t *pa[N];</span></a>
|
|
<a name="2629"><span class="lineNum"> 2629 </span><span class="lineCov"> 43770 : uint8_t G[N * N];</span></a>
|
|
<a name="2630"><span class="lineNum"> 2630 </span><span class="lineCov"> 43770 : uint8_t V[N * N];</span></a>
|
|
<a name="2631"><span class="lineNum"> 2631 </span> : size_t i;</a>
|
|
<a name="2632"><span class="lineNum"> 2632 </span> : int j, k;</a>
|
|
<a name="2633"><span class="lineNum"> 2633 </span> : </a>
|
|
<a name="2634"><span class="lineNum"> 2634 </span> : (void)nr; /* unused, it's always 2 */</a>
|
|
<a name="2635"><span class="lineNum"> 2635 </span> : </a>
|
|
<a name="2636"><span class="lineNum"> 2636 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="2637"><span class="lineNum"> 2637 </span><span class="lineCov"> 131310 : for (j = 0; j < N; ++j)</span></a>
|
|
<a name="2638"><span class="lineNum"> 2638 </span><span class="lineCov"> 262620 : for (k = 0; k < N; ++k)</span></a>
|
|
<a name="2639"><span class="lineNum"> 2639 </span><span class="lineCov"> 350160 : G[j * N + k] = A(ip[j], id[k]);</span></a>
|
|
<a name="2640"><span class="lineNum"> 2640 </span> : </a>
|
|
<a name="2641"><span class="lineNum"> 2641 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="2642"><span class="lineNum"> 2642 </span><span class="lineCov"> 43770 : raid_invert(G, V, N);</span></a>
|
|
<a name="2643"><span class="lineNum"> 2643 </span> : </a>
|
|
<a name="2644"><span class="lineNum"> 2644 </span> : /* compute delta parity */</a>
|
|
<a name="2645"><span class="lineNum"> 2645 </span><span class="lineCov"> 43770 : raid_delta_gen(N, id, ip, nd, size, vv);</span></a>
|
|
<a name="2646"><span class="lineNum"> 2646 </span> : </a>
|
|
<a name="2647"><span class="lineNum"> 2647 </span><span class="lineCov"> 131310 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2648"><span class="lineNum"> 2648 </span><span class="lineCov"> 87540 : p[j] = v[nd + ip[j]];</span></a>
|
|
<a name="2649"><span class="lineNum"> 2649 </span><span class="lineCov"> 87540 : pa[j] = v[id[j]];</span></a>
|
|
<a name="2650"><span class="lineNum"> 2650 </span> : }</a>
|
|
<a name="2651"><span class="lineNum"> 2651 </span> : </a>
|
|
<a name="2652"><span class="lineNum"> 2652 </span> : raid_sse_begin();</a>
|
|
<a name="2653"><span class="lineNum"> 2653 </span> : </a>
|
|
<a name="2654"><span class="lineNum"> 2654 </span><span class="lineCov"> 43770 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2655"><span class="lineNum"> 2655 </span> : </a>
|
|
<a name="2656"><span class="lineNum"> 2656 </span><span class="lineCov"> 2918586 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="2657"><span class="lineNum"> 2657 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));</span></a>
|
|
<a name="2658"><span class="lineNum"> 2658 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm2" : : "m" (pa[0][i]));</span></a>
|
|
<a name="2659"><span class="lineNum"> 2659 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));</span></a>
|
|
<a name="2660"><span class="lineNum"> 2660 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm3" : : "m" (pa[1][i]));</span></a>
|
|
<a name="2661"><span class="lineNum"> 2661 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm2,%xmm0");</span></a>
|
|
<a name="2662"><span class="lineNum"> 2662 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm3,%xmm1");</span></a>
|
|
<a name="2663"><span class="lineNum"> 2663 </span> : </a>
|
|
<a name="2664"><span class="lineNum"> 2664 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm6,%xmm6");</span></a>
|
|
<a name="2665"><span class="lineNum"> 2665 </span> : </a>
|
|
<a name="2666"><span class="lineNum"> 2666 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[0]][0][0]));</span></a>
|
|
<a name="2667"><span class="lineNum"> 2667 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[0]][1][0]));</span></a>
|
|
<a name="2668"><span class="lineNum"> 2668 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm0,%xmm4");</span></a>
|
|
<a name="2669"><span class="lineNum"> 2669 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm0,%xmm5");</span></a>
|
|
<a name="2670"><span class="lineNum"> 2670 </span><span class="lineCov"> 2874816 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="2671"><span class="lineNum"> 2671 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2672"><span class="lineNum"> 2672 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="2673"><span class="lineNum"> 2673 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="2674"><span class="lineNum"> 2674 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm5,%xmm3");</span></a>
|
|
<a name="2675"><span class="lineNum"> 2675 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm2,%xmm6");</span></a>
|
|
<a name="2676"><span class="lineNum"> 2676 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm3,%xmm6");</span></a>
|
|
<a name="2677"><span class="lineNum"> 2677 </span> : </a>
|
|
<a name="2678"><span class="lineNum"> 2678 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[1]][0][0]));</span></a>
|
|
<a name="2679"><span class="lineNum"> 2679 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[1]][1][0]));</span></a>
|
|
<a name="2680"><span class="lineNum"> 2680 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm1,%xmm4");</span></a>
|
|
<a name="2681"><span class="lineNum"> 2681 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm1,%xmm5");</span></a>
|
|
<a name="2682"><span class="lineNum"> 2682 </span><span class="lineCov"> 2874816 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="2683"><span class="lineNum"> 2683 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2684"><span class="lineNum"> 2684 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="2685"><span class="lineNum"> 2685 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="2686"><span class="lineNum"> 2686 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm5,%xmm3");</span></a>
|
|
<a name="2687"><span class="lineNum"> 2687 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm2,%xmm6");</span></a>
|
|
<a name="2688"><span class="lineNum"> 2688 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm3,%xmm6");</span></a>
|
|
<a name="2689"><span class="lineNum"> 2689 </span> : </a>
|
|
<a name="2690"><span class="lineNum"> 2690 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[0][i]));</span></a>
|
|
<a name="2691"><span class="lineNum"> 2691 </span> : </a>
|
|
<a name="2692"><span class="lineNum"> 2692 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm6,%xmm6");</span></a>
|
|
<a name="2693"><span class="lineNum"> 2693 </span> : </a>
|
|
<a name="2694"><span class="lineNum"> 2694 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[2]][0][0]));</span></a>
|
|
<a name="2695"><span class="lineNum"> 2695 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[2]][1][0]));</span></a>
|
|
<a name="2696"><span class="lineNum"> 2696 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm0,%xmm4");</span></a>
|
|
<a name="2697"><span class="lineNum"> 2697 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm0,%xmm5");</span></a>
|
|
<a name="2698"><span class="lineNum"> 2698 </span><span class="lineCov"> 2874816 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="2699"><span class="lineNum"> 2699 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2700"><span class="lineNum"> 2700 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="2701"><span class="lineNum"> 2701 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="2702"><span class="lineNum"> 2702 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm5,%xmm3");</span></a>
|
|
<a name="2703"><span class="lineNum"> 2703 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm2,%xmm6");</span></a>
|
|
<a name="2704"><span class="lineNum"> 2704 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm3,%xmm6");</span></a>
|
|
<a name="2705"><span class="lineNum"> 2705 </span> : </a>
|
|
<a name="2706"><span class="lineNum"> 2706 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[V[3]][0][0]));</span></a>
|
|
<a name="2707"><span class="lineNum"> 2707 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[V[3]][1][0]));</span></a>
|
|
<a name="2708"><span class="lineNum"> 2708 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm1,%xmm4");</span></a>
|
|
<a name="2709"><span class="lineNum"> 2709 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %xmm1,%xmm5");</span></a>
|
|
<a name="2710"><span class="lineNum"> 2710 </span><span class="lineCov"> 2874816 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="2711"><span class="lineNum"> 2711 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2712"><span class="lineNum"> 2712 </span><span class="lineCov"> 2874816 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="2713"><span class="lineNum"> 2713 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="2714"><span class="lineNum"> 2714 </span><span class="lineCov"> 2874816 : asm volatile ("pshufb %xmm5,%xmm3");</span></a>
|
|
<a name="2715"><span class="lineNum"> 2715 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm2,%xmm6");</span></a>
|
|
<a name="2716"><span class="lineNum"> 2716 </span><span class="lineCov"> 2874816 : asm volatile ("pxor %xmm3,%xmm6");</span></a>
|
|
<a name="2717"><span class="lineNum"> 2717 </span> : </a>
|
|
<a name="2718"><span class="lineNum"> 2718 </span><span class="lineCov"> 2874816 : asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));</span></a>
|
|
<a name="2719"><span class="lineNum"> 2719 </span> : }</a>
|
|
<a name="2720"><span class="lineNum"> 2720 </span> : </a>
|
|
<a name="2721"><span class="lineNum"> 2721 </span> : raid_sse_end();</a>
|
|
<a name="2722"><span class="lineNum"> 2722 </span><span class="lineCov"> 43770 : }</span></a>
|
|
<a name="2723"><span class="lineNum"> 2723 </span> : #endif</a>
|
|
<a name="2724"><span class="lineNum"> 2724 </span> : </a>
|
|
<a name="2725"><span class="lineNum"> 2725 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSSE3)</a>
|
|
<a name="2726"><span class="lineNum"> 2726 </span> : /*</a>
|
|
<a name="2727"><span class="lineNum"> 2727 </span> : * RAID recovering SSSE3 implementation</a>
|
|
<a name="2728"><span class="lineNum"> 2728 </span> : */</a>
|
|
<a name="2729"><span class="lineNum"> 2729 </span><span class="lineCov"> 92519 : void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="2730"><span class="lineNum"> 2730 </span> : {</a>
|
|
<a name="2731"><span class="lineNum"> 2731 </span><span class="lineCov"> 92519 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2732"><span class="lineNum"> 2732 </span><span class="lineCov"> 92519 : int N = nr;</span></a>
|
|
<a name="2733"><span class="lineNum"> 2733 </span> : uint8_t *p[RAID_PARITY_MAX];</a>
|
|
<a name="2734"><span class="lineNum"> 2734 </span> : uint8_t *pa[RAID_PARITY_MAX];</a>
|
|
<a name="2735"><span class="lineNum"> 2735 </span> : uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];</a>
|
|
<a name="2736"><span class="lineNum"> 2736 </span> : uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];</a>
|
|
<a name="2737"><span class="lineNum"> 2737 </span> : uint8_t buffer[RAID_PARITY_MAX*16+16];</a>
|
|
<a name="2738"><span class="lineNum"> 2738 </span><span class="lineCov"> 92519 : uint8_t *pd = __align_ptr(buffer, 16);</span></a>
|
|
<a name="2739"><span class="lineNum"> 2739 </span> : size_t i;</a>
|
|
<a name="2740"><span class="lineNum"> 2740 </span> : int j, k;</a>
|
|
<a name="2741"><span class="lineNum"> 2741 </span> : </a>
|
|
<a name="2742"><span class="lineNum"> 2742 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="2743"><span class="lineNum"> 2743 </span><span class="lineCov"> 487519 : for (j = 0; j < N; ++j)</span></a>
|
|
<a name="2744"><span class="lineNum"> 2744 </span><span class="lineCov"> 2200562 : for (k = 0; k < N; ++k)</span></a>
|
|
<a name="2745"><span class="lineNum"> 2745 </span><span class="lineCov"> 3611124 : G[j * N + k] = A(ip[j], id[k]);</span></a>
|
|
<a name="2746"><span class="lineNum"> 2746 </span> : </a>
|
|
<a name="2747"><span class="lineNum"> 2747 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="2748"><span class="lineNum"> 2748 </span><span class="lineCov"> 92519 : raid_invert(G, V, N);</span></a>
|
|
<a name="2749"><span class="lineNum"> 2749 </span> : </a>
|
|
<a name="2750"><span class="lineNum"> 2750 </span> : /* compute delta parity */</a>
|
|
<a name="2751"><span class="lineNum"> 2751 </span><span class="lineCov"> 92519 : raid_delta_gen(N, id, ip, nd, size, vv);</span></a>
|
|
<a name="2752"><span class="lineNum"> 2752 </span> : </a>
|
|
<a name="2753"><span class="lineNum"> 2753 </span><span class="lineCov"> 487519 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2754"><span class="lineNum"> 2754 </span><span class="lineCov"> 395000 : p[j] = v[nd + ip[j]];</span></a>
|
|
<a name="2755"><span class="lineNum"> 2755 </span><span class="lineCov"> 395000 : pa[j] = v[id[j]];</span></a>
|
|
<a name="2756"><span class="lineNum"> 2756 </span> : }</a>
|
|
<a name="2757"><span class="lineNum"> 2757 </span> : </a>
|
|
<a name="2758"><span class="lineNum"> 2758 </span> : raid_sse_begin();</a>
|
|
<a name="2759"><span class="lineNum"> 2759 </span> : </a>
|
|
<a name="2760"><span class="lineNum"> 2760 </span><span class="lineCov"> 92519 : asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2761"><span class="lineNum"> 2761 </span> : </a>
|
|
<a name="2762"><span class="lineNum"> 2762 </span><span class="lineCov"> 5505847 : for (i = 0; i < size; i += 16) {</span></a>
|
|
<a name="2763"><span class="lineNum"> 2763 </span> : /* delta */</a>
|
|
<a name="2764"><span class="lineNum"> 2764 </span><span class="lineCov"> 28632016 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2765"><span class="lineNum"> 2765 </span><span class="lineCov"> 23218688 : asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i]));</span></a>
|
|
<a name="2766"><span class="lineNum"> 2766 </span><span class="lineCov"> 23218688 : asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[j][i]));</span></a>
|
|
<a name="2767"><span class="lineNum"> 2767 </span><span class="lineCov"> 23218688 : asm volatile ("pxor %xmm1,%xmm0");</span></a>
|
|
<a name="2768"><span class="lineNum"> 2768 </span><span class="lineCov"> 23218688 : asm volatile ("movdqa %%xmm0,%0" : "=m" (pd[j*16]));</span></a>
|
|
<a name="2769"><span class="lineNum"> 2769 </span> : }</a>
|
|
<a name="2770"><span class="lineNum"> 2770 </span> : </a>
|
|
<a name="2771"><span class="lineNum"> 2771 </span> : /* reconstruct */</a>
|
|
<a name="2772"><span class="lineNum"> 2772 </span><span class="lineCov"> 28632016 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2773"><span class="lineNum"> 2773 </span><span class="lineCov"> 23218688 : asm volatile ("pxor %xmm0,%xmm0");</span></a>
|
|
<a name="2774"><span class="lineNum"> 2774 </span><span class="lineCov"> 23218688 : asm volatile ("pxor %xmm1,%xmm1");</span></a>
|
|
<a name="2775"><span class="lineNum"> 2775 </span> : </a>
|
|
<a name="2776"><span class="lineNum"> 2776 </span><span class="lineCov"> 130240064 : for (k = 0; k < N; ++k) {</span></a>
|
|
<a name="2777"><span class="lineNum"> 2777 </span><span class="lineCov"> 107021376 : uint8_t m = V[j * N + k];</span></a>
|
|
<a name="2778"><span class="lineNum"> 2778 </span> : </a>
|
|
<a name="2779"><span class="lineNum"> 2779 </span><span class="lineCov"> 107021376 : asm volatile ("movdqa %0,%%xmm2" : : "m" (gfmulpshufb[m][0][0]));</span></a>
|
|
<a name="2780"><span class="lineNum"> 2780 </span><span class="lineCov"> 107021376 : asm volatile ("movdqa %0,%%xmm3" : : "m" (gfmulpshufb[m][1][0]));</span></a>
|
|
<a name="2781"><span class="lineNum"> 2781 </span><span class="lineCov"> 107021376 : asm volatile ("movdqa %0,%%xmm4" : : "m" (pd[k*16]));</span></a>
|
|
<a name="2782"><span class="lineNum"> 2782 </span><span class="lineCov"> 107021376 : asm volatile ("movdqa %xmm4,%xmm5");</span></a>
|
|
<a name="2783"><span class="lineNum"> 2783 </span><span class="lineCov"> 107021376 : asm volatile ("psrlw $4,%xmm5");</span></a>
|
|
<a name="2784"><span class="lineNum"> 2784 </span><span class="lineCov"> 107021376 : asm volatile ("pand %xmm7,%xmm4");</span></a>
|
|
<a name="2785"><span class="lineNum"> 2785 </span><span class="lineCov"> 107021376 : asm volatile ("pand %xmm7,%xmm5");</span></a>
|
|
<a name="2786"><span class="lineNum"> 2786 </span><span class="lineCov"> 107021376 : asm volatile ("pshufb %xmm4,%xmm2");</span></a>
|
|
<a name="2787"><span class="lineNum"> 2787 </span><span class="lineCov"> 107021376 : asm volatile ("pshufb %xmm5,%xmm3");</span></a>
|
|
<a name="2788"><span class="lineNum"> 2788 </span><span class="lineCov"> 107021376 : asm volatile ("pxor %xmm2,%xmm0");</span></a>
|
|
<a name="2789"><span class="lineNum"> 2789 </span><span class="lineCov"> 107021376 : asm volatile ("pxor %xmm3,%xmm1");</span></a>
|
|
<a name="2790"><span class="lineNum"> 2790 </span> : }</a>
|
|
<a name="2791"><span class="lineNum"> 2791 </span> : </a>
|
|
<a name="2792"><span class="lineNum"> 2792 </span><span class="lineCov"> 23218688 : asm volatile ("pxor %xmm1,%xmm0");</span></a>
|
|
<a name="2793"><span class="lineNum"> 2793 </span><span class="lineCov"> 23218688 : asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));</span></a>
|
|
<a name="2794"><span class="lineNum"> 2794 </span> : }</a>
|
|
<a name="2795"><span class="lineNum"> 2795 </span> : }</a>
|
|
<a name="2796"><span class="lineNum"> 2796 </span> : </a>
|
|
<a name="2797"><span class="lineNum"> 2797 </span> : raid_sse_end();</a>
|
|
<a name="2798"><span class="lineNum"> 2798 </span><span class="lineCov"> 92519 : }</span></a>
|
|
<a name="2799"><span class="lineNum"> 2799 </span> : #endif</a>
|
|
<a name="2800"><span class="lineNum"> 2800 </span> : </a>
|
|
<a name="2801"><span class="lineNum"> 2801 </span> : #if defined(CONFIG_X86) && defined(CONFIG_AVX2)</a>
|
|
<a name="2802"><span class="lineNum"> 2802 </span> : /*</a>
|
|
<a name="2803"><span class="lineNum"> 2803 </span> : * RAID recovering for one disk AVX2 implementation</a>
|
|
<a name="2804"><span class="lineNum"> 2804 </span> : */</a>
|
|
<a name="2805"><span class="lineNum"> 2805 </span><span class="lineCov"> 114 : void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="2806"><span class="lineNum"> 2806 </span> : {</a>
|
|
<a name="2807"><span class="lineNum"> 2807 </span><span class="lineCov"> 114 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2808"><span class="lineNum"> 2808 </span> : uint8_t *p;</a>
|
|
<a name="2809"><span class="lineNum"> 2809 </span> : uint8_t *pa;</a>
|
|
<a name="2810"><span class="lineNum"> 2810 </span> : uint8_t G;</a>
|
|
<a name="2811"><span class="lineNum"> 2811 </span> : uint8_t V;</a>
|
|
<a name="2812"><span class="lineNum"> 2812 </span> : size_t i;</a>
|
|
<a name="2813"><span class="lineNum"> 2813 </span> : </a>
|
|
<a name="2814"><span class="lineNum"> 2814 </span> : (void)nr; /* unused, it's always 1 */</a>
|
|
<a name="2815"><span class="lineNum"> 2815 </span> : </a>
|
|
<a name="2816"><span class="lineNum"> 2816 </span> : /* if it's RAID5 uses the faster function */</a>
|
|
<a name="2817"><span class="lineNum"> 2817 </span><span class="lineCov"> 114 : if (ip[0] == 0) {</span></a>
|
|
<a name="2818"><span class="lineNum"> 2818 </span><span class="lineCov"> 24 : raid_rec1of1(id, nd, size, vv);</span></a>
|
|
<a name="2819"><span class="lineNum"> 2819 </span><span class="lineCov"> 24 : return;</span></a>
|
|
<a name="2820"><span class="lineNum"> 2820 </span> : }</a>
|
|
<a name="2821"><span class="lineNum"> 2821 </span> : </a>
|
|
<a name="2822"><span class="lineNum"> 2822 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="2823"><span class="lineNum"> 2823 </span><span class="lineCov"> 90 : G = A(ip[0], id[0]);</span></a>
|
|
<a name="2824"><span class="lineNum"> 2824 </span> : </a>
|
|
<a name="2825"><span class="lineNum"> 2825 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="2826"><span class="lineNum"> 2826 </span><span class="lineCov"> 90 : V = inv(G);</span></a>
|
|
<a name="2827"><span class="lineNum"> 2827 </span> : </a>
|
|
<a name="2828"><span class="lineNum"> 2828 </span> : /* compute delta parity */</a>
|
|
<a name="2829"><span class="lineNum"> 2829 </span><span class="lineCov"> 90 : raid_delta_gen(1, id, ip, nd, size, vv);</span></a>
|
|
<a name="2830"><span class="lineNum"> 2830 </span> : </a>
|
|
<a name="2831"><span class="lineNum"> 2831 </span><span class="lineCov"> 90 : p = v[nd + ip[0]];</span></a>
|
|
<a name="2832"><span class="lineNum"> 2832 </span><span class="lineCov"> 90 : pa = v[id[0]];</span></a>
|
|
<a name="2833"><span class="lineNum"> 2833 </span> : </a>
|
|
<a name="2834"><span class="lineNum"> 2834 </span> : raid_avx_begin();</a>
|
|
<a name="2835"><span class="lineNum"> 2835 </span> : </a>
|
|
<a name="2836"><span class="lineNum"> 2836 </span><span class="lineCov"> 90 : asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2837"><span class="lineNum"> 2837 </span><span class="lineCov"> 90 : asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0]));</span></a>
|
|
<a name="2838"><span class="lineNum"> 2838 </span><span class="lineCov"> 90 : asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfmulpshufb[V][1][0]));</span></a>
|
|
<a name="2839"><span class="lineNum"> 2839 </span> : </a>
|
|
<a name="2840"><span class="lineNum"> 2840 </span><span class="lineCov"> 49914 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="2841"><span class="lineNum"> 2841 </span><span class="lineCov"> 49824 : asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));</span></a>
|
|
<a name="2842"><span class="lineNum"> 2842 </span><span class="lineCov"> 49824 : asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[i]));</span></a>
|
|
<a name="2843"><span class="lineNum"> 2843 </span><span class="lineCov"> 49824 : asm volatile ("vpxor %ymm1,%ymm0,%ymm0");</span></a>
|
|
<a name="2844"><span class="lineNum"> 2844 </span><span class="lineCov"> 49824 : asm volatile ("vpsrlw $4,%ymm0,%ymm1");</span></a>
|
|
<a name="2845"><span class="lineNum"> 2845 </span><span class="lineCov"> 49824 : asm volatile ("vpand %ymm7,%ymm0,%ymm0");</span></a>
|
|
<a name="2846"><span class="lineNum"> 2846 </span><span class="lineCov"> 49824 : asm volatile ("vpand %ymm7,%ymm1,%ymm1");</span></a>
|
|
<a name="2847"><span class="lineNum"> 2847 </span><span class="lineCov"> 49824 : asm volatile ("vpshufb %ymm0,%ymm4,%ymm2");</span></a>
|
|
<a name="2848"><span class="lineNum"> 2848 </span><span class="lineCov"> 49824 : asm volatile ("vpshufb %ymm1,%ymm5,%ymm3");</span></a>
|
|
<a name="2849"><span class="lineNum"> 2849 </span><span class="lineCov"> 49824 : asm volatile ("vpxor %ymm3,%ymm2,%ymm2");</span></a>
|
|
<a name="2850"><span class="lineNum"> 2850 </span><span class="lineCov"> 49824 : asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i]));</span></a>
|
|
<a name="2851"><span class="lineNum"> 2851 </span> : }</a>
|
|
<a name="2852"><span class="lineNum"> 2852 </span> : </a>
|
|
<a name="2853"><span class="lineNum"> 2853 </span> : raid_avx_end();</a>
|
|
<a name="2854"><span class="lineNum"> 2854 </span> : }</a>
|
|
<a name="2855"><span class="lineNum"> 2855 </span> : #endif</a>
|
|
<a name="2856"><span class="lineNum"> 2856 </span> : </a>
|
|
<a name="2857"><span class="lineNum"> 2857 </span> : #if defined(CONFIG_X86) && defined(CONFIG_AVX2)</a>
|
|
<a name="2858"><span class="lineNum"> 2858 </span> : /*</a>
|
|
<a name="2859"><span class="lineNum"> 2859 </span> : * RAID recovering for two disks AVX2 implementation</a>
|
|
<a name="2860"><span class="lineNum"> 2860 </span> : */</a>
|
|
<a name="2861"><span class="lineNum"> 2861 </span><span class="lineCov"> 1190 : void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="2862"><span class="lineNum"> 2862 </span><span class="lineCov"> 1190 : {</span></a>
|
|
<a name="2863"><span class="lineNum"> 2863 </span><span class="lineCov"> 1190 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2864"><span class="lineNum"> 2864 </span><span class="lineCov"> 1190 : const int N = 2;</span></a>
|
|
<a name="2865"><span class="lineNum"> 2865 </span><span class="lineCov"> 1190 : uint8_t *p[N];</span></a>
|
|
<a name="2866"><span class="lineNum"> 2866 </span><span class="lineCov"> 1190 : uint8_t *pa[N];</span></a>
|
|
<a name="2867"><span class="lineNum"> 2867 </span><span class="lineCov"> 1190 : uint8_t G[N * N];</span></a>
|
|
<a name="2868"><span class="lineNum"> 2868 </span><span class="lineCov"> 1190 : uint8_t V[N * N];</span></a>
|
|
<a name="2869"><span class="lineNum"> 2869 </span> : size_t i;</a>
|
|
<a name="2870"><span class="lineNum"> 2870 </span> : int j, k;</a>
|
|
<a name="2871"><span class="lineNum"> 2871 </span> : </a>
|
|
<a name="2872"><span class="lineNum"> 2872 </span> : (void)nr; /* unused, it's always 2 */</a>
|
|
<a name="2873"><span class="lineNum"> 2873 </span> : </a>
|
|
<a name="2874"><span class="lineNum"> 2874 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="2875"><span class="lineNum"> 2875 </span><span class="lineCov"> 3570 : for (j = 0; j < N; ++j)</span></a>
|
|
<a name="2876"><span class="lineNum"> 2876 </span><span class="lineCov"> 7140 : for (k = 0; k < N; ++k)</span></a>
|
|
<a name="2877"><span class="lineNum"> 2877 </span><span class="lineCov"> 9520 : G[j * N + k] = A(ip[j], id[k]);</span></a>
|
|
<a name="2878"><span class="lineNum"> 2878 </span> : </a>
|
|
<a name="2879"><span class="lineNum"> 2879 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="2880"><span class="lineNum"> 2880 </span><span class="lineCov"> 1190 : raid_invert(G, V, N);</span></a>
|
|
<a name="2881"><span class="lineNum"> 2881 </span> : </a>
|
|
<a name="2882"><span class="lineNum"> 2882 </span> : /* compute delta parity */</a>
|
|
<a name="2883"><span class="lineNum"> 2883 </span><span class="lineCov"> 1190 : raid_delta_gen(N, id, ip, nd, size, vv);</span></a>
|
|
<a name="2884"><span class="lineNum"> 2884 </span> : </a>
|
|
<a name="2885"><span class="lineNum"> 2885 </span><span class="lineCov"> 3570 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2886"><span class="lineNum"> 2886 </span><span class="lineCov"> 2380 : p[j] = v[nd + ip[j]];</span></a>
|
|
<a name="2887"><span class="lineNum"> 2887 </span><span class="lineCov"> 2380 : pa[j] = v[id[j]];</span></a>
|
|
<a name="2888"><span class="lineNum"> 2888 </span> : }</a>
|
|
<a name="2889"><span class="lineNum"> 2889 </span> : </a>
|
|
<a name="2890"><span class="lineNum"> 2890 </span> : raid_avx_begin();</a>
|
|
<a name="2891"><span class="lineNum"> 2891 </span> : </a>
|
|
<a name="2892"><span class="lineNum"> 2892 </span><span class="lineCov"> 1190 : asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2893"><span class="lineNum"> 2893 </span> : </a>
|
|
<a name="2894"><span class="lineNum"> 2894 </span><span class="lineCov"> 27078 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="2895"><span class="lineNum"> 2895 </span><span class="lineCov"> 25888 : asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));</span></a>
|
|
<a name="2896"><span class="lineNum"> 2896 </span><span class="lineCov"> 25888 : asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[0][i]));</span></a>
|
|
<a name="2897"><span class="lineNum"> 2897 </span><span class="lineCov"> 25888 : asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i]));</span></a>
|
|
<a name="2898"><span class="lineNum"> 2898 </span><span class="lineCov"> 25888 : asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[1][i]));</span></a>
|
|
<a name="2899"><span class="lineNum"> 2899 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm2,%ymm0,%ymm0");</span></a>
|
|
<a name="2900"><span class="lineNum"> 2900 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm3,%ymm1,%ymm1");</span></a>
|
|
<a name="2901"><span class="lineNum"> 2901 </span> : </a>
|
|
<a name="2902"><span class="lineNum"> 2902 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm6,%ymm6,%ymm6");</span></a>
|
|
<a name="2903"><span class="lineNum"> 2903 </span> : </a>
|
|
<a name="2904"><span class="lineNum"> 2904 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[0]][0][0]));</span></a>
|
|
<a name="2905"><span class="lineNum"> 2905 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[0]][1][0]));</span></a>
|
|
<a name="2906"><span class="lineNum"> 2906 </span><span class="lineCov"> 25888 : asm volatile ("vpsrlw $4,%ymm0,%ymm5");</span></a>
|
|
<a name="2907"><span class="lineNum"> 2907 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm0,%ymm4");</span></a>
|
|
<a name="2908"><span class="lineNum"> 2908 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="2909"><span class="lineNum"> 2909 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="2910"><span class="lineNum"> 2910 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="2911"><span class="lineNum"> 2911 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm2,%ymm6,%ymm6");</span></a>
|
|
<a name="2912"><span class="lineNum"> 2912 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm3,%ymm6,%ymm6");</span></a>
|
|
<a name="2913"><span class="lineNum"> 2913 </span> : </a>
|
|
<a name="2914"><span class="lineNum"> 2914 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[1]][0][0]));</span></a>
|
|
<a name="2915"><span class="lineNum"> 2915 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[1]][1][0]));</span></a>
|
|
<a name="2916"><span class="lineNum"> 2916 </span><span class="lineCov"> 25888 : asm volatile ("vpsrlw $4,%ymm1,%ymm5");</span></a>
|
|
<a name="2917"><span class="lineNum"> 2917 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm1,%ymm4");</span></a>
|
|
<a name="2918"><span class="lineNum"> 2918 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="2919"><span class="lineNum"> 2919 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="2920"><span class="lineNum"> 2920 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="2921"><span class="lineNum"> 2921 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm2,%ymm6,%ymm6");</span></a>
|
|
<a name="2922"><span class="lineNum"> 2922 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm3,%ymm6,%ymm6");</span></a>
|
|
<a name="2923"><span class="lineNum"> 2923 </span> : </a>
|
|
<a name="2924"><span class="lineNum"> 2924 </span><span class="lineCov"> 25888 : asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[0][i]));</span></a>
|
|
<a name="2925"><span class="lineNum"> 2925 </span> : </a>
|
|
<a name="2926"><span class="lineNum"> 2926 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm6,%ymm6,%ymm6");</span></a>
|
|
<a name="2927"><span class="lineNum"> 2927 </span> : </a>
|
|
<a name="2928"><span class="lineNum"> 2928 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[2]][0][0]));</span></a>
|
|
<a name="2929"><span class="lineNum"> 2929 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[2]][1][0]));</span></a>
|
|
<a name="2930"><span class="lineNum"> 2930 </span><span class="lineCov"> 25888 : asm volatile ("vpsrlw $4,%ymm0,%ymm5");</span></a>
|
|
<a name="2931"><span class="lineNum"> 2931 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm0,%ymm4");</span></a>
|
|
<a name="2932"><span class="lineNum"> 2932 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="2933"><span class="lineNum"> 2933 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="2934"><span class="lineNum"> 2934 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="2935"><span class="lineNum"> 2935 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm2,%ymm6,%ymm6");</span></a>
|
|
<a name="2936"><span class="lineNum"> 2936 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm3,%ymm6,%ymm6");</span></a>
|
|
<a name="2937"><span class="lineNum"> 2937 </span> : </a>
|
|
<a name="2938"><span class="lineNum"> 2938 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[V[3]][0][0]));</span></a>
|
|
<a name="2939"><span class="lineNum"> 2939 </span><span class="lineCov"> 25888 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[V[3]][1][0]));</span></a>
|
|
<a name="2940"><span class="lineNum"> 2940 </span><span class="lineCov"> 25888 : asm volatile ("vpsrlw $4,%ymm1,%ymm5");</span></a>
|
|
<a name="2941"><span class="lineNum"> 2941 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm1,%ymm4");</span></a>
|
|
<a name="2942"><span class="lineNum"> 2942 </span><span class="lineCov"> 25888 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="2943"><span class="lineNum"> 2943 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="2944"><span class="lineNum"> 2944 </span><span class="lineCov"> 25888 : asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="2945"><span class="lineNum"> 2945 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm2,%ymm6,%ymm6");</span></a>
|
|
<a name="2946"><span class="lineNum"> 2946 </span><span class="lineCov"> 25888 : asm volatile ("vpxor %ymm3,%ymm6,%ymm6");</span></a>
|
|
<a name="2947"><span class="lineNum"> 2947 </span> : </a>
|
|
<a name="2948"><span class="lineNum"> 2948 </span><span class="lineCov"> 25888 : asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));</span></a>
|
|
<a name="2949"><span class="lineNum"> 2949 </span> : }</a>
|
|
<a name="2950"><span class="lineNum"> 2950 </span> : </a>
|
|
<a name="2951"><span class="lineNum"> 2951 </span> : raid_avx_end();</a>
|
|
<a name="2952"><span class="lineNum"> 2952 </span><span class="lineCov"> 1190 : }</span></a>
|
|
<a name="2953"><span class="lineNum"> 2953 </span> : #endif</a>
|
|
<a name="2954"><span class="lineNum"> 2954 </span> : </a>
|
|
<a name="2955"><span class="lineNum"> 2955 </span> : #if defined(CONFIG_X86) && defined(CONFIG_AVX2)</a>
|
|
<a name="2956"><span class="lineNum"> 2956 </span> : /*</a>
|
|
<a name="2957"><span class="lineNum"> 2957 </span> : * RAID recovering AVX2 implementation</a>
|
|
<a name="2958"><span class="lineNum"> 2958 </span> : */</a>
|
|
<a name="2959"><span class="lineNum"> 2959 </span><span class="lineCov"> 17729 : void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="2960"><span class="lineNum"> 2960 </span> : {</a>
|
|
<a name="2961"><span class="lineNum"> 2961 </span><span class="lineCov"> 17729 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="2962"><span class="lineNum"> 2962 </span><span class="lineCov"> 17729 : int N = nr;</span></a>
|
|
<a name="2963"><span class="lineNum"> 2963 </span> : uint8_t *p[RAID_PARITY_MAX];</a>
|
|
<a name="2964"><span class="lineNum"> 2964 </span> : uint8_t *pa[RAID_PARITY_MAX];</a>
|
|
<a name="2965"><span class="lineNum"> 2965 </span> : uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];</a>
|
|
<a name="2966"><span class="lineNum"> 2966 </span> : uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];</a>
|
|
<a name="2967"><span class="lineNum"> 2967 </span> : uint8_t buffer[RAID_PARITY_MAX*32+32];</a>
|
|
<a name="2968"><span class="lineNum"> 2968 </span><span class="lineCov"> 17729 : uint8_t *pd = __align_ptr(buffer, 32);</span></a>
|
|
<a name="2969"><span class="lineNum"> 2969 </span> : size_t i;</a>
|
|
<a name="2970"><span class="lineNum"> 2970 </span> : int j, k;</a>
|
|
<a name="2971"><span class="lineNum"> 2971 </span> : </a>
|
|
<a name="2972"><span class="lineNum"> 2972 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="2973"><span class="lineNum"> 2973 </span><span class="lineCov"> 90629 : for (j = 0; j < N; ++j)</span></a>
|
|
<a name="2974"><span class="lineNum"> 2974 </span><span class="lineCov"> 385516 : for (k = 0; k < N; ++k)</span></a>
|
|
<a name="2975"><span class="lineNum"> 2975 </span><span class="lineCov"> 625232 : G[j * N + k] = A(ip[j], id[k]);</span></a>
|
|
<a name="2976"><span class="lineNum"> 2976 </span> : </a>
|
|
<a name="2977"><span class="lineNum"> 2977 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="2978"><span class="lineNum"> 2978 </span><span class="lineCov"> 17729 : raid_invert(G, V, N);</span></a>
|
|
<a name="2979"><span class="lineNum"> 2979 </span> : </a>
|
|
<a name="2980"><span class="lineNum"> 2980 </span> : /* compute delta parity */</a>
|
|
<a name="2981"><span class="lineNum"> 2981 </span><span class="lineCov"> 17729 : raid_delta_gen(N, id, ip, nd, size, vv);</span></a>
|
|
<a name="2982"><span class="lineNum"> 2982 </span> : </a>
|
|
<a name="2983"><span class="lineNum"> 2983 </span><span class="lineCov"> 90629 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2984"><span class="lineNum"> 2984 </span><span class="lineCov"> 72900 : p[j] = v[nd + ip[j]];</span></a>
|
|
<a name="2985"><span class="lineNum"> 2985 </span><span class="lineCov"> 72900 : pa[j] = v[id[j]];</span></a>
|
|
<a name="2986"><span class="lineNum"> 2986 </span> : }</a>
|
|
<a name="2987"><span class="lineNum"> 2987 </span> : </a>
|
|
<a name="2988"><span class="lineNum"> 2988 </span> : raid_avx_begin();</a>
|
|
<a name="2989"><span class="lineNum"> 2989 </span> : </a>
|
|
<a name="2990"><span class="lineNum"> 2990 </span><span class="lineCov"> 17729 : asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="2991"><span class="lineNum"> 2991 </span> : </a>
|
|
<a name="2992"><span class="lineNum"> 2992 </span><span class="lineCov"> 225033 : for (i = 0; i < size; i += 32) {</span></a>
|
|
<a name="2993"><span class="lineNum"> 2993 </span> : /* delta */</a>
|
|
<a name="2994"><span class="lineNum"> 2994 </span><span class="lineCov"> 1085128 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="2995"><span class="lineNum"> 2995 </span><span class="lineCov"> 877824 : asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i]));</span></a>
|
|
<a name="2996"><span class="lineNum"> 2996 </span><span class="lineCov"> 877824 : asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[j][i]));</span></a>
|
|
<a name="2997"><span class="lineNum"> 2997 </span><span class="lineCov"> 877824 : asm volatile ("vpxor %ymm1,%ymm0,%ymm0");</span></a>
|
|
<a name="2998"><span class="lineNum"> 2998 </span><span class="lineCov"> 877824 : asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j*32]));</span></a>
|
|
<a name="2999"><span class="lineNum"> 2999 </span> : }</a>
|
|
<a name="3000"><span class="lineNum"> 3000 </span> : </a>
|
|
<a name="3001"><span class="lineNum"> 3001 </span> : /* reconstruct */</a>
|
|
<a name="3002"><span class="lineNum"> 3002 </span><span class="lineCov"> 1085128 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="3003"><span class="lineNum"> 3003 </span><span class="lineCov"> 877824 : asm volatile ("vpxor %ymm0,%ymm0,%ymm0");</span></a>
|
|
<a name="3004"><span class="lineNum"> 3004 </span><span class="lineCov"> 877824 : asm volatile ("vpxor %ymm1,%ymm1,%ymm1");</span></a>
|
|
<a name="3005"><span class="lineNum"> 3005 </span> : </a>
|
|
<a name="3006"><span class="lineNum"> 3006 </span><span class="lineCov"> 4786400 : for (k = 0; k < N; ++k) {</span></a>
|
|
<a name="3007"><span class="lineNum"> 3007 </span><span class="lineCov"> 3908576 : uint8_t m = V[j * N + k];</span></a>
|
|
<a name="3008"><span class="lineNum"> 3008 </span> : </a>
|
|
<a name="3009"><span class="lineNum"> 3009 </span><span class="lineCov"> 3908576 : asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (gfmulpshufb[m][0][0]));</span></a>
|
|
<a name="3010"><span class="lineNum"> 3010 </span><span class="lineCov"> 3908576 : asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfmulpshufb[m][1][0]));</span></a>
|
|
<a name="3011"><span class="lineNum"> 3011 </span><span class="lineCov"> 3908576 : asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k*32]));</span></a>
|
|
<a name="3012"><span class="lineNum"> 3012 </span><span class="lineCov"> 3908576 : asm volatile ("vpsrlw $4,%ymm4,%ymm5");</span></a>
|
|
<a name="3013"><span class="lineNum"> 3013 </span><span class="lineCov"> 3908576 : asm volatile ("vpand %ymm7,%ymm4,%ymm4");</span></a>
|
|
<a name="3014"><span class="lineNum"> 3014 </span><span class="lineCov"> 3908576 : asm volatile ("vpand %ymm7,%ymm5,%ymm5");</span></a>
|
|
<a name="3015"><span class="lineNum"> 3015 </span><span class="lineCov"> 3908576 : asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");</span></a>
|
|
<a name="3016"><span class="lineNum"> 3016 </span><span class="lineCov"> 3908576 : asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");</span></a>
|
|
<a name="3017"><span class="lineNum"> 3017 </span><span class="lineCov"> 3908576 : asm volatile ("vpxor %ymm2,%ymm0,%ymm0");</span></a>
|
|
<a name="3018"><span class="lineNum"> 3018 </span><span class="lineCov"> 3908576 : asm volatile ("vpxor %ymm3,%ymm1,%ymm1");</span></a>
|
|
<a name="3019"><span class="lineNum"> 3019 </span> : }</a>
|
|
<a name="3020"><span class="lineNum"> 3020 </span> : </a>
|
|
<a name="3021"><span class="lineNum"> 3021 </span><span class="lineCov"> 877824 : asm volatile ("vpxor %ymm1,%ymm0,%ymm0");</span></a>
|
|
<a name="3022"><span class="lineNum"> 3022 </span><span class="lineCov"> 877824 : asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));</span></a>
|
|
<a name="3023"><span class="lineNum"> 3023 </span> : }</a>
|
|
<a name="3024"><span class="lineNum"> 3024 </span> : }</a>
|
|
<a name="3025"><span class="lineNum"> 3025 </span> : </a>
|
|
<a name="3026"><span class="lineNum"> 3026 </span> : raid_avx_end();</a>
|
|
<a name="3027"><span class="lineNum"> 3027 </span><span class="lineCov"> 17729 : }</span></a>
|
|
<a name="3028"><span class="lineNum"> 3028 </span> : #endif</a>
|
|
<a name="3029"><span class="lineNum"> 3029 </span> : </a>
|
|
<a name="3030"><span class="lineNum"> 3030 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="3031"><span class="lineNum"> 3031 </span> : /*</a>
|
|
<a name="3032"><span class="lineNum"> 3032 </span> : * RAID recovering for one disk AVX512BW implementation</a>
|
|
<a name="3033"><span class="lineNum"> 3033 </span> : */</a>
|
|
<a name="3034"><span class="lineNum"> 3034 </span><span class="lineCov"> 121 : void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="3035"><span class="lineNum"> 3035 </span> : {</a>
|
|
<a name="3036"><span class="lineNum"> 3036 </span><span class="lineCov"> 121 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="3037"><span class="lineNum"> 3037 </span> : uint8_t *p, *pa;</a>
|
|
<a name="3038"><span class="lineNum"> 3038 </span> : uint8_t G, V;</a>
|
|
<a name="3039"><span class="lineNum"> 3039 </span> : size_t i;</a>
|
|
<a name="3040"><span class="lineNum"> 3040 </span> : </a>
|
|
<a name="3041"><span class="lineNum"> 3041 </span> : (void)nr;</a>
|
|
<a name="3042"><span class="lineNum"> 3042 </span> : </a>
|
|
<a name="3043"><span class="lineNum"> 3043 </span><span class="lineCov"> 121 : if (ip[0] == 0) {</span></a>
|
|
<a name="3044"><span class="lineNum"> 3044 </span><span class="lineCov"> 30 : raid_rec1of1(id, nd, size, vv);</span></a>
|
|
<a name="3045"><span class="lineNum"> 3045 </span><span class="lineCov"> 30 : return;</span></a>
|
|
<a name="3046"><span class="lineNum"> 3046 </span> : }</a>
|
|
<a name="3047"><span class="lineNum"> 3047 </span> : </a>
|
|
<a name="3048"><span class="lineNum"> 3048 </span><span class="lineCov"> 91 : G = A(ip[0], id[0]);</span></a>
|
|
<a name="3049"><span class="lineNum"> 3049 </span><span class="lineCov"> 91 : V = inv(G);</span></a>
|
|
<a name="3050"><span class="lineNum"> 3050 </span> : </a>
|
|
<a name="3051"><span class="lineNum"> 3051 </span><span class="lineCov"> 91 : raid_delta_gen(1, id, ip, nd, size, vv);</span></a>
|
|
<a name="3052"><span class="lineNum"> 3052 </span> : </a>
|
|
<a name="3053"><span class="lineNum"> 3053 </span><span class="lineCov"> 91 : p = v[nd + ip[0]];</span></a>
|
|
<a name="3054"><span class="lineNum"> 3054 </span><span class="lineCov"> 91 : pa = v[id[0]];</span></a>
|
|
<a name="3055"><span class="lineNum"> 3055 </span> : </a>
|
|
<a name="3056"><span class="lineNum"> 3056 </span> : raid_avx_begin();</a>
|
|
<a name="3057"><span class="lineNum"> 3057 </span> : </a>
|
|
<a name="3058"><span class="lineNum"> 3058 </span><span class="lineCov"> 91 : asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="3059"><span class="lineNum"> 3059 </span><span class="lineCov"> 91 : asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfmulpshufb[V][0][0]));</span></a>
|
|
<a name="3060"><span class="lineNum"> 3060 </span><span class="lineCov"> 91 : asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfmulpshufb[V][1][0]));</span></a>
|
|
<a name="3061"><span class="lineNum"> 3061 </span> : </a>
|
|
<a name="3062"><span class="lineNum"> 3062 </span><span class="lineCov"> 12971 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="3063"><span class="lineNum"> 3063 </span><span class="lineCov"> 12880 : asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));</span></a>
|
|
<a name="3064"><span class="lineNum"> 3064 </span><span class="lineCov"> 12880 : asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));</span></a>
|
|
<a name="3065"><span class="lineNum"> 3065 </span><span class="lineCov"> 12880 : asm volatile ("vpxord %zmm1,%zmm0,%zmm0");</span></a>
|
|
<a name="3066"><span class="lineNum"> 3066 </span><span class="lineCov"> 12880 : asm volatile ("vpsrlw $4,%zmm0,%zmm1");</span></a>
|
|
<a name="3067"><span class="lineNum"> 3067 </span><span class="lineCov"> 12880 : asm volatile ("vpandd %zmm7,%zmm0,%zmm0");</span></a>
|
|
<a name="3068"><span class="lineNum"> 3068 </span><span class="lineCov"> 12880 : asm volatile ("vpandd %zmm7,%zmm1,%zmm1");</span></a>
|
|
<a name="3069"><span class="lineNum"> 3069 </span><span class="lineCov"> 12880 : asm volatile ("vpshufb %zmm0,%zmm4,%zmm2");</span></a>
|
|
<a name="3070"><span class="lineNum"> 3070 </span><span class="lineCov"> 12880 : asm volatile ("vpshufb %zmm1,%zmm5,%zmm3");</span></a>
|
|
<a name="3071"><span class="lineNum"> 3071 </span><span class="lineCov"> 12880 : asm volatile ("vpxord %zmm3,%zmm2,%zmm2");</span></a>
|
|
<a name="3072"><span class="lineNum"> 3072 </span><span class="lineCov"> 12880 : asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i]));</span></a>
|
|
<a name="3073"><span class="lineNum"> 3073 </span> : }</a>
|
|
<a name="3074"><span class="lineNum"> 3074 </span> : </a>
|
|
<a name="3075"><span class="lineNum"> 3075 </span> : raid_avx_end();</a>
|
|
<a name="3076"><span class="lineNum"> 3076 </span> : }</a>
|
|
<a name="3077"><span class="lineNum"> 3077 </span> : #endif</a>
|
|
<a name="3078"><span class="lineNum"> 3078 </span> : </a>
|
|
<a name="3079"><span class="lineNum"> 3079 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="3080"><span class="lineNum"> 3080 </span> : /*</a>
|
|
<a name="3081"><span class="lineNum"> 3081 </span> : * RAID recovering for two disks AVX512BW implementation</a>
|
|
<a name="3082"><span class="lineNum"> 3082 </span> : */</a>
|
|
<a name="3083"><span class="lineNum"> 3083 </span><span class="lineCov"> 1199 : void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="3084"><span class="lineNum"> 3084 </span><span class="lineCov"> 1199 : {</span></a>
|
|
<a name="3085"><span class="lineNum"> 3085 </span><span class="lineCov"> 1199 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="3086"><span class="lineNum"> 3086 </span><span class="lineCov"> 1199 : const int N = 2;</span></a>
|
|
<a name="3087"><span class="lineNum"> 3087 </span><span class="lineCov"> 1199 : uint8_t *p[N], *pa[N];</span></a>
|
|
<a name="3088"><span class="lineNum"> 3088 </span><span class="lineCov"> 1199 : uint8_t G[N * N], V[N * N];</span></a>
|
|
<a name="3089"><span class="lineNum"> 3089 </span> : size_t i;</a>
|
|
<a name="3090"><span class="lineNum"> 3090 </span> : int j, k;</a>
|
|
<a name="3091"><span class="lineNum"> 3091 </span> : </a>
|
|
<a name="3092"><span class="lineNum"> 3092 </span> : (void)nr;</a>
|
|
<a name="3093"><span class="lineNum"> 3093 </span> : </a>
|
|
<a name="3094"><span class="lineNum"> 3094 </span><span class="lineCov"> 3597 : for (j = 0; j < N; ++j)</span></a>
|
|
<a name="3095"><span class="lineNum"> 3095 </span><span class="lineCov"> 7194 : for (k = 0; k < N; ++k)</span></a>
|
|
<a name="3096"><span class="lineNum"> 3096 </span><span class="lineCov"> 9592 : G[j * N + k] = A(ip[j], id[k]);</span></a>
|
|
<a name="3097"><span class="lineNum"> 3097 </span> : </a>
|
|
<a name="3098"><span class="lineNum"> 3098 </span><span class="lineCov"> 1199 : raid_invert(G, V, N);</span></a>
|
|
<a name="3099"><span class="lineNum"> 3099 </span><span class="lineCov"> 1199 : raid_delta_gen(N, id, ip, nd, size, vv);</span></a>
|
|
<a name="3100"><span class="lineNum"> 3100 </span> : </a>
|
|
<a name="3101"><span class="lineNum"> 3101 </span><span class="lineCov"> 3597 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="3102"><span class="lineNum"> 3102 </span><span class="lineCov"> 2398 : p[j] = v[nd + ip[j]];</span></a>
|
|
<a name="3103"><span class="lineNum"> 3103 </span><span class="lineCov"> 2398 : pa[j] = v[id[j]];</span></a>
|
|
<a name="3104"><span class="lineNum"> 3104 </span> : }</a>
|
|
<a name="3105"><span class="lineNum"> 3105 </span> : </a>
|
|
<a name="3106"><span class="lineNum"> 3106 </span> : raid_avx_begin();</a>
|
|
<a name="3107"><span class="lineNum"> 3107 </span> : </a>
|
|
<a name="3108"><span class="lineNum"> 3108 </span><span class="lineCov"> 1199 : asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="3109"><span class="lineNum"> 3109 </span> : </a>
|
|
<a name="3110"><span class="lineNum"> 3110 </span><span class="lineCov"> 10687 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="3111"><span class="lineNum"> 3111 </span><span class="lineCov"> 9488 : asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));</span></a>
|
|
<a name="3112"><span class="lineNum"> 3112 </span><span class="lineCov"> 9488 : asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));</span></a>
|
|
<a name="3113"><span class="lineNum"> 3113 </span><span class="lineCov"> 9488 : asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));</span></a>
|
|
<a name="3114"><span class="lineNum"> 3114 </span><span class="lineCov"> 9488 : asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));</span></a>
|
|
<a name="3115"><span class="lineNum"> 3115 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm2,%zmm0,%zmm0");</span></a>
|
|
<a name="3116"><span class="lineNum"> 3116 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm3,%zmm1,%zmm1");</span></a>
|
|
<a name="3117"><span class="lineNum"> 3117 </span> : </a>
|
|
<a name="3118"><span class="lineNum"> 3118 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="3119"><span class="lineNum"> 3119 </span> : </a>
|
|
<a name="3120"><span class="lineNum"> 3120 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[0]][0][0]));</span></a>
|
|
<a name="3121"><span class="lineNum"> 3121 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[0]][1][0]));</span></a>
|
|
<a name="3122"><span class="lineNum"> 3122 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[1]][0][0]));</span></a>
|
|
<a name="3123"><span class="lineNum"> 3123 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[1]][1][0]));</span></a>
|
|
<a name="3124"><span class="lineNum"> 3124 </span><span class="lineCov"> 9488 : asm volatile ("vpsrlw $4,%zmm0,%zmm5");</span></a>
|
|
<a name="3125"><span class="lineNum"> 3125 </span><span class="lineCov"> 9488 : asm volatile ("vpsrlw $4,%zmm1,%zmm13");</span></a>
|
|
<a name="3126"><span class="lineNum"> 3126 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm0,%zmm4");</span></a>
|
|
<a name="3127"><span class="lineNum"> 3127 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm5,%zmm5");</span></a>
|
|
<a name="3128"><span class="lineNum"> 3128 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm1,%zmm12");</span></a>
|
|
<a name="3129"><span class="lineNum"> 3129 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm13,%zmm13");</span></a>
|
|
<a name="3130"><span class="lineNum"> 3130 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm4,%zmm2,%zmm2");</span></a>
|
|
<a name="3131"><span class="lineNum"> 3131 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm5,%zmm3,%zmm3");</span></a>
|
|
<a name="3132"><span class="lineNum"> 3132 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm12,%zmm10,%zmm10");</span></a>
|
|
<a name="3133"><span class="lineNum"> 3133 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm13,%zmm11,%zmm11");</span></a>
|
|
<a name="3134"><span class="lineNum"> 3134 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm2,%zmm6,%zmm6");</span></a>
|
|
<a name="3135"><span class="lineNum"> 3135 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm3,%zmm6,%zmm6");</span></a>
|
|
<a name="3136"><span class="lineNum"> 3136 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm10,%zmm6,%zmm6");</span></a>
|
|
<a name="3137"><span class="lineNum"> 3137 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm11,%zmm6,%zmm6");</span></a>
|
|
<a name="3138"><span class="lineNum"> 3138 </span> : </a>
|
|
<a name="3139"><span class="lineNum"> 3139 </span><span class="lineCov"> 9488 : asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i]));</span></a>
|
|
<a name="3140"><span class="lineNum"> 3140 </span> : </a>
|
|
<a name="3141"><span class="lineNum"> 3141 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm6,%zmm6,%zmm6");</span></a>
|
|
<a name="3142"><span class="lineNum"> 3142 </span> : </a>
|
|
<a name="3143"><span class="lineNum"> 3143 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[V[2]][0][0]));</span></a>
|
|
<a name="3144"><span class="lineNum"> 3144 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[V[2]][1][0]));</span></a>
|
|
<a name="3145"><span class="lineNum"> 3145 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm10" : : "m" (gfmulpshufb[V[3]][0][0]));</span></a>
|
|
<a name="3146"><span class="lineNum"> 3146 </span><span class="lineCov"> 9488 : asm volatile ("vbroadcasti32x4 %0,%%zmm11" : : "m" (gfmulpshufb[V[3]][1][0]));</span></a>
|
|
<a name="3147"><span class="lineNum"> 3147 </span><span class="lineCov"> 9488 : asm volatile ("vpsrlw $4,%zmm0,%zmm5");</span></a>
|
|
<a name="3148"><span class="lineNum"> 3148 </span><span class="lineCov"> 9488 : asm volatile ("vpsrlw $4,%zmm1,%zmm13");</span></a>
|
|
<a name="3149"><span class="lineNum"> 3149 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm0,%zmm4");</span></a>
|
|
<a name="3150"><span class="lineNum"> 3150 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm5,%zmm5");</span></a>
|
|
<a name="3151"><span class="lineNum"> 3151 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm1,%zmm12");</span></a>
|
|
<a name="3152"><span class="lineNum"> 3152 </span><span class="lineCov"> 9488 : asm volatile ("vpandd %zmm7,%zmm13,%zmm13");</span></a>
|
|
<a name="3153"><span class="lineNum"> 3153 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm4,%zmm2,%zmm2");</span></a>
|
|
<a name="3154"><span class="lineNum"> 3154 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm5,%zmm3,%zmm3");</span></a>
|
|
<a name="3155"><span class="lineNum"> 3155 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm12,%zmm10,%zmm10");</span></a>
|
|
<a name="3156"><span class="lineNum"> 3156 </span><span class="lineCov"> 9488 : asm volatile ("vpshufb %zmm13,%zmm11,%zmm11");</span></a>
|
|
<a name="3157"><span class="lineNum"> 3157 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm2,%zmm6,%zmm6");</span></a>
|
|
<a name="3158"><span class="lineNum"> 3158 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm3,%zmm6,%zmm6");</span></a>
|
|
<a name="3159"><span class="lineNum"> 3159 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm10,%zmm6,%zmm6");</span></a>
|
|
<a name="3160"><span class="lineNum"> 3160 </span><span class="lineCov"> 9488 : asm volatile ("vpxord %zmm11,%zmm6,%zmm6");</span></a>
|
|
<a name="3161"><span class="lineNum"> 3161 </span> : </a>
|
|
<a name="3162"><span class="lineNum"> 3162 </span><span class="lineCov"> 9488 : asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i]));</span></a>
|
|
<a name="3163"><span class="lineNum"> 3163 </span> : }</a>
|
|
<a name="3164"><span class="lineNum"> 3164 </span> : </a>
|
|
<a name="3165"><span class="lineNum"> 3165 </span> : raid_avx_end();</a>
|
|
<a name="3166"><span class="lineNum"> 3166 </span><span class="lineCov"> 1199 : }</span></a>
|
|
<a name="3167"><span class="lineNum"> 3167 </span> : #endif</a>
|
|
<a name="3168"><span class="lineNum"> 3168 </span> : </a>
|
|
<a name="3169"><span class="lineNum"> 3169 </span> : #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512BW)</a>
|
|
<a name="3170"><span class="lineNum"> 3170 </span> : /*</a>
|
|
<a name="3171"><span class="lineNum"> 3171 </span> : * RAID recovering AVX512BW implementation</a>
|
|
<a name="3172"><span class="lineNum"> 3172 </span> : */</a>
|
|
<a name="3173"><span class="lineNum"> 3173 </span><span class="lineCov"> 17737 : void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span></a>
|
|
<a name="3174"><span class="lineNum"> 3174 </span> : {</a>
|
|
<a name="3175"><span class="lineNum"> 3175 </span><span class="lineCov"> 17737 : uint8_t **v = (uint8_t **)vv;</span></a>
|
|
<a name="3176"><span class="lineNum"> 3176 </span><span class="lineCov"> 17737 : int N = nr;</span></a>
|
|
<a name="3177"><span class="lineNum"> 3177 </span> : uint8_t *p[RAID_PARITY_MAX];</a>
|
|
<a name="3178"><span class="lineNum"> 3178 </span> : uint8_t *pa[RAID_PARITY_MAX];</a>
|
|
<a name="3179"><span class="lineNum"> 3179 </span> : uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];</a>
|
|
<a name="3180"><span class="lineNum"> 3180 </span> : uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];</a>
|
|
<a name="3181"><span class="lineNum"> 3181 </span> : uint8_t buffer[RAID_PARITY_MAX*64+64];</a>
|
|
<a name="3182"><span class="lineNum"> 3182 </span><span class="lineCov"> 17737 : uint8_t *pd = __align_ptr(buffer, 64);</span></a>
|
|
<a name="3183"><span class="lineNum"> 3183 </span> : size_t i;</a>
|
|
<a name="3184"><span class="lineNum"> 3184 </span> : int j, k;</a>
|
|
<a name="3185"><span class="lineNum"> 3185 </span> : </a>
|
|
<a name="3186"><span class="lineNum"> 3186 </span> : /* setup the coefficients matrix */</a>
|
|
<a name="3187"><span class="lineNum"> 3187 </span><span class="lineCov"> 90667 : for (j = 0; j < N; ++j)</span></a>
|
|
<a name="3188"><span class="lineNum"> 3188 </span><span class="lineCov"> 385668 : for (k = 0; k < N; ++k)</span></a>
|
|
<a name="3189"><span class="lineNum"> 3189 </span><span class="lineCov"> 625476 : G[j * N + k] = A(ip[j], id[k]);</span></a>
|
|
<a name="3190"><span class="lineNum"> 3190 </span> : </a>
|
|
<a name="3191"><span class="lineNum"> 3191 </span> : /* invert it to solve the system of linear equations */</a>
|
|
<a name="3192"><span class="lineNum"> 3192 </span><span class="lineCov"> 17737 : raid_invert(G, V, N);</span></a>
|
|
<a name="3193"><span class="lineNum"> 3193 </span> : </a>
|
|
<a name="3194"><span class="lineNum"> 3194 </span> : /* compute delta parity */</a>
|
|
<a name="3195"><span class="lineNum"> 3195 </span><span class="lineCov"> 17737 : raid_delta_gen(N, id, ip, nd, size, vv);</span></a>
|
|
<a name="3196"><span class="lineNum"> 3196 </span> : </a>
|
|
<a name="3197"><span class="lineNum"> 3197 </span><span class="lineCov"> 90667 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="3198"><span class="lineNum"> 3198 </span><span class="lineCov"> 72930 : p[j] = v[nd + ip[j]];</span></a>
|
|
<a name="3199"><span class="lineNum"> 3199 </span><span class="lineCov"> 72930 : pa[j] = v[id[j]];</span></a>
|
|
<a name="3200"><span class="lineNum"> 3200 </span> : }</a>
|
|
<a name="3201"><span class="lineNum"> 3201 </span> : </a>
|
|
<a name="3202"><span class="lineNum"> 3202 </span> : raid_avx_begin();</a>
|
|
<a name="3203"><span class="lineNum"> 3203 </span> : </a>
|
|
<a name="3204"><span class="lineNum"> 3204 </span><span class="lineCov"> 17737 : asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0]));</span></a>
|
|
<a name="3205"><span class="lineNum"> 3205 </span> : </a>
|
|
<a name="3206"><span class="lineNum"> 3206 </span><span class="lineCov"> 105773 : for (i = 0; i < size; i += 64) {</span></a>
|
|
<a name="3207"><span class="lineNum"> 3207 </span> : /* delta */</a>
|
|
<a name="3208"><span class="lineNum"> 3208 </span><span class="lineCov"> 456292 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="3209"><span class="lineNum"> 3209 </span><span class="lineCov"> 368256 : asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i]));</span></a>
|
|
<a name="3210"><span class="lineNum"> 3210 </span><span class="lineCov"> 368256 : asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i]));</span></a>
|
|
<a name="3211"><span class="lineNum"> 3211 </span><span class="lineCov"> 368256 : asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");</span></a>
|
|
<a name="3212"><span class="lineNum"> 3212 </span><span class="lineCov"> 368256 : asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64]));</span></a>
|
|
<a name="3213"><span class="lineNum"> 3213 </span> : }</a>
|
|
<a name="3214"><span class="lineNum"> 3214 </span> : /* reconstruct */</a>
|
|
<a name="3215"><span class="lineNum"> 3215 </span><span class="lineCov"> 456292 : for (j = 0; j < N; ++j) {</span></a>
|
|
<a name="3216"><span class="lineNum"> 3216 </span><span class="lineCov"> 368256 : asm volatile ("vpxorq %zmm0,%zmm0,%zmm0");</span></a>
|
|
<a name="3217"><span class="lineNum"> 3217 </span><span class="lineCov"> 368256 : asm volatile ("vpxorq %zmm1,%zmm1,%zmm1");</span></a>
|
|
<a name="3218"><span class="lineNum"> 3218 </span><span class="lineCov"> 1983600 : for (k = 0; k < N; ++k) {</span></a>
|
|
<a name="3219"><span class="lineNum"> 3219 </span><span class="lineCov"> 1615344 : uint8_t m = V[j * N + k];</span></a>
|
|
<a name="3220"><span class="lineNum"> 3220 </span><span class="lineCov"> 1615344 : asm volatile ("vbroadcasti32x4 %0,%%zmm2" : : "m" (gfmulpshufb[m][0][0]));</span></a>
|
|
<a name="3221"><span class="lineNum"> 3221 </span><span class="lineCov"> 1615344 : asm volatile ("vbroadcasti32x4 %0,%%zmm3" : : "m" (gfmulpshufb[m][1][0]));</span></a>
|
|
<a name="3222"><span class="lineNum"> 3222 </span><span class="lineCov"> 1615344 : asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64]));</span></a>
|
|
<a name="3223"><span class="lineNum"> 3223 </span><span class="lineCov"> 1615344 : asm volatile ("vpsrlw $4,%zmm4,%zmm5");</span></a>
|
|
<a name="3224"><span class="lineNum"> 3224 </span><span class="lineCov"> 1615344 : asm volatile ("vpandq %zmm7,%zmm4,%zmm4");</span></a>
|
|
<a name="3225"><span class="lineNum"> 3225 </span><span class="lineCov"> 1615344 : asm volatile ("vpandq %zmm7,%zmm5,%zmm5");</span></a>
|
|
<a name="3226"><span class="lineNum"> 3226 </span><span class="lineCov"> 1615344 : asm volatile ("vpshufb %zmm4,%zmm2,%zmm2");</span></a>
|
|
<a name="3227"><span class="lineNum"> 3227 </span><span class="lineCov"> 1615344 : asm volatile ("vpshufb %zmm5,%zmm3,%zmm3");</span></a>
|
|
<a name="3228"><span class="lineNum"> 3228 </span><span class="lineCov"> 1615344 : asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");</span></a>
|
|
<a name="3229"><span class="lineNum"> 3229 </span><span class="lineCov"> 1615344 : asm volatile ("vpxorq %zmm3,%zmm1,%zmm1");</span></a>
|
|
<a name="3230"><span class="lineNum"> 3230 </span> : }</a>
|
|
<a name="3231"><span class="lineNum"> 3231 </span><span class="lineCov"> 368256 : asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");</span></a>
|
|
<a name="3232"><span class="lineNum"> 3232 </span><span class="lineCov"> 368256 : asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i]));</span></a>
|
|
<a name="3233"><span class="lineNum"> 3233 </span> : }</a>
|
|
<a name="3234"><span class="lineNum"> 3234 </span> : }</a>
|
|
<a name="3235"><span class="lineNum"> 3235 </span> : </a>
|
|
<a name="3236"><span class="lineNum"> 3236 </span> : raid_avx_end();</a>
|
|
<a name="3237"><span class="lineNum"> 3237 </span><span class="lineCov"> 17737 : }</span></a>
|
|
<a name="3238"><span class="lineNum"> 3238 </span> : #endif</a>
|
|
<a name="3239"><span class="lineNum"> 3239 </span> : </a>
|
|
<a name="3240"><span class="lineNum"> 3240 </span> : #if defined(CONFIG_X86) && defined(CONFIG_SSE2)</a>
|
|
<a name="3241"><span class="lineNum"> 3241 </span><span class="lineCov"> 322 : void raid_register_x86(void)</span></a>
|
|
<a name="3242"><span class="lineNum"> 3242 </span> : {</a>
|
|
<a name="3243"><span class="lineNum"> 3243 </span> : #ifdef CONFIG_SSE2</a>
|
|
<a name="3244"><span class="lineNum"> 3244 </span><span class="lineCov"> 322 : if (raid_cpu_has_sse2()) {</span></a>
|
|
<a name="3245"><span class="lineNum"> 3245 </span><span class="lineCov"> 322 : raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2);</span></a>
|
|
<a name="3246"><span class="lineNum"> 3246 </span><span class="lineCov"> 322 : raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2);</span></a>
|
|
<a name="3247"><span class="lineNum"> 3247 </span> : #ifdef CONFIG_X86_64</a>
|
|
<a name="3248"><span class="lineNum"> 3248 </span><span class="lineCov"> 322 : if (!raid_cpu_has_slowextendedreg())</span></a>
|
|
<a name="3249"><span class="lineNum"> 3249 </span><span class="lineCov"> 322 : raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext);</span></a>
|
|
<a name="3250"><span class="lineNum"> 3250 </span> : /* note that raid_cpu_has_slowextendedreg() doesn't affect vandermonde */</a>
|
|
<a name="3251"><span class="lineNum"> 3251 </span><span class="lineCov"> 322 : raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext);</span></a>
|
|
<a name="3252"><span class="lineNum"> 3252 </span> : #else</a>
|
|
<a name="3253"><span class="lineNum"> 3253 </span> : raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2", raid_genz_sse2);</a>
|
|
<a name="3254"><span class="lineNum"> 3254 </span> : #endif</a>
|
|
<a name="3255"><span class="lineNum"> 3255 </span> : }</a>
|
|
<a name="3256"><span class="lineNum"> 3256 </span> : #endif</a>
|
|
<a name="3257"><span class="lineNum"> 3257 </span> : </a>
|
|
<a name="3258"><span class="lineNum"> 3258 </span> : #ifdef CONFIG_SSSE3</a>
|
|
<a name="3259"><span class="lineNum"> 3259 </span><span class="lineCov"> 322 : if (raid_cpu_has_ssse3()) {</span></a>
|
|
<a name="3260"><span class="lineNum"> 3260 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3);</span></a>
|
|
<a name="3261"><span class="lineNum"> 3261 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3);</span></a>
|
|
<a name="3262"><span class="lineNum"> 3262 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3);</span></a>
|
|
<a name="3263"><span class="lineNum"> 3263 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3);</span></a>
|
|
<a name="3264"><span class="lineNum"> 3264 </span> : #ifdef CONFIG_X86_64</a>
|
|
<a name="3265"><span class="lineNum"> 3265 </span><span class="lineCov"> 321 : if (!raid_cpu_has_slowextendedreg()) {</span></a>
|
|
<a name="3266"><span class="lineNum"> 3266 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext);</span></a>
|
|
<a name="3267"><span class="lineNum"> 3267 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext);</span></a>
|
|
<a name="3268"><span class="lineNum"> 3268 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext);</span></a>
|
|
<a name="3269"><span class="lineNum"> 3269 </span><span class="lineCov"> 321 : raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext);</span></a>
|
|
<a name="3270"><span class="lineNum"> 3270 </span> : }</a>
|
|
<a name="3271"><span class="lineNum"> 3271 </span> : #endif</a>
|
|
<a name="3272"><span class="lineNum"> 3272 </span> : </a>
|
|
<a name="3273"><span class="lineNum"> 3273 </span><span class="lineCov"> 321 : raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3);</span></a>
|
|
<a name="3274"><span class="lineNum"> 3274 </span><span class="lineCov"> 321 : raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3);</span></a>
|
|
<a name="3275"><span class="lineNum"> 3275 </span><span class="lineCov"> 321 : raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3);</span></a>
|
|
<a name="3276"><span class="lineNum"> 3276 </span><span class="lineCov"> 321 : raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3);</span></a>
|
|
<a name="3277"><span class="lineNum"> 3277 </span><span class="lineCov"> 321 : raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3);</span></a>
|
|
<a name="3278"><span class="lineNum"> 3278 </span><span class="lineCov"> 321 : raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3);</span></a>
|
|
<a name="3279"><span class="lineNum"> 3279 </span> : }</a>
|
|
<a name="3280"><span class="lineNum"> 3280 </span> : #endif</a>
|
|
<a name="3281"><span class="lineNum"> 3281 </span> : </a>
|
|
<a name="3282"><span class="lineNum"> 3282 </span> : #ifdef CONFIG_AVX2</a>
|
|
<a name="3283"><span class="lineNum"> 3283 </span><span class="lineCov"> 322 : if (raid_cpu_has_avx2()) {</span></a>
|
|
<a name="3284"><span class="lineNum"> 3284 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2);</span></a>
|
|
<a name="3285"><span class="lineNum"> 3285 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2);</span></a>
|
|
<a name="3286"><span class="lineNum"> 3286 </span> : #ifdef CONFIG_X86_64</a>
|
|
<a name="3287"><span class="lineNum"> 3287 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext);</span></a>
|
|
<a name="3288"><span class="lineNum"> 3288 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext);</span></a>
|
|
<a name="3289"><span class="lineNum"> 3289 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext);</span></a>
|
|
<a name="3290"><span class="lineNum"> 3290 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext);</span></a>
|
|
<a name="3291"><span class="lineNum"> 3291 </span><span class="lineCov"> 3 : raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext);</span></a>
|
|
<a name="3292"><span class="lineNum"> 3292 </span> : #endif</a>
|
|
<a name="3293"><span class="lineNum"> 3293 </span> : </a>
|
|
<a name="3294"><span class="lineNum"> 3294 </span><span class="lineCov"> 3 : raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2);</span></a>
|
|
<a name="3295"><span class="lineNum"> 3295 </span><span class="lineCov"> 3 : raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2);</span></a>
|
|
<a name="3296"><span class="lineNum"> 3296 </span><span class="lineCov"> 3 : raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2);</span></a>
|
|
<a name="3297"><span class="lineNum"> 3297 </span><span class="lineCov"> 3 : raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2);</span></a>
|
|
<a name="3298"><span class="lineNum"> 3298 </span><span class="lineCov"> 3 : raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2);</span></a>
|
|
<a name="3299"><span class="lineNum"> 3299 </span><span class="lineCov"> 3 : raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2);</span></a>
|
|
<a name="3300"><span class="lineNum"> 3300 </span> : }</a>
|
|
<a name="3301"><span class="lineNum"> 3301 </span> : #endif</a>
|
|
<a name="3302"><span class="lineNum"> 3302 </span> : </a>
|
|
<a name="3303"><span class="lineNum"> 3303 </span> : #ifdef CONFIG_AVX512BW</a>
|
|
<a name="3304"><span class="lineNum"> 3304 </span> : #ifdef CONFIG_X86_64</a>
|
|
<a name="3305"><span class="lineNum"> 3305 </span><span class="lineCov"> 322 : if (raid_cpu_has_avx512bw()) {</span></a>
|
|
<a name="3306"><span class="lineNum"> 3306 </span><span class="lineCov"> 2 : raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_gen1_avx512bw);</span></a>
|
|
<a name="3307"><span class="lineNum"> 3307 </span><span class="lineCov"> 2 : raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_gen2_avx512bw);</span></a>
|
|
<a name="3308"><span class="lineNum"> 3308 </span><span class="lineCov"> 2 : raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_gen3_avx512bw);</span></a>
|
|
<a name="3309"><span class="lineNum"> 3309 </span><span class="lineCov"> 2 : raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_gen4_avx512bw);</span></a>
|
|
<a name="3310"><span class="lineNum"> 3310 </span><span class="lineCov"> 2 : raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_gen5_avx512bw);</span></a>
|
|
<a name="3311"><span class="lineNum"> 3311 </span><span class="lineCov"> 2 : raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_gen6_avx512bw);</span></a>
|
|
<a name="3312"><span class="lineNum"> 3312 </span> : </a>
|
|
<a name="3313"><span class="lineNum"> 3313 </span><span class="lineCov"> 2 : raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw);</span></a>
|
|
<a name="3314"><span class="lineNum"> 3314 </span><span class="lineCov"> 2 : raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw);</span></a>
|
|
<a name="3315"><span class="lineNum"> 3315 </span><span class="lineCov"> 2 : raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw);</span></a>
|
|
<a name="3316"><span class="lineNum"> 3316 </span><span class="lineCov"> 2 : raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw);</span></a>
|
|
<a name="3317"><span class="lineNum"> 3317 </span><span class="lineCov"> 2 : raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw);</span></a>
|
|
<a name="3318"><span class="lineNum"> 3318 </span><span class="lineCov"> 2 : raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw);</span></a>
|
|
<a name="3319"><span class="lineNum"> 3319 </span> : }</a>
|
|
<a name="3320"><span class="lineNum"> 3320 </span> : #endif</a>
|
|
<a name="3321"><span class="lineNum"> 3321 </span> : #endif</a>
|
|
<a name="3322"><span class="lineNum"> 3322 </span><span class="lineCov"> 322 : }</span></a>
|
|
<a name="3323"><span class="lineNum"> 3323 </span> : #endif</a>
|
|
</pre>
|
|
</td>
|
|
</tr>
|
|
</table>
|
|
<br>
|
|
|
|
<table width="100%" border=0 cellspacing=0 cellpadding=0>
|
|
<tr><td class="ruler"><img src="../glass.png" width=3 height=3 alt=""></td></tr>
|
|
<tr><td class="versionInfo">Generated by: <a href="https://github.com/linux-test-project/lcov" target="_parent">LCOV version 1.0</a></td></tr>
|
|
</table>
|
|
<br>
|
|
|
|
</body>
|
|
</html>
|