snapraid/cov/raid/x86.c.gcov.html
Andrea Mazzoleni e01ca1e0fa Auto update
2016-11-21 19:48:21 +01:00

2528 lines
267 KiB
HTML

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>LCOV - lcov.info - raid/x86.c</title>
<link rel="stylesheet" type="text/css" href="../gcov.css">
</head>
<body>
<table width="100%" border=0 cellspacing=0 cellpadding=0>
<tr><td class="title">LCOV - code coverage report</td></tr>
<tr><td class="ruler"><img src="../glass.png" width=3 height=3 alt=""></td></tr>
<tr>
<td width="100%">
<table cellpadding=1 border=0 width="100%">
<tr>
<td width="10%" class="headerItem">Current view:</td>
<td width="35%" class="headerValue"><a href="../index.html">top level</a> - <a href="index.html">raid</a> - x86.c<span style="font-size: 80%;"> (source / <a href="x86.c.func.html">functions</a>)</span></td>
<td width="5%"></td>
<td width="15%"></td>
<td width="10%" class="headerCovTableHead">Hit</td>
<td width="10%" class="headerCovTableHead">Total</td>
<td width="15%" class="headerCovTableHead">Coverage</td>
</tr>
<tr>
<td class="headerItem">Test:</td>
<td class="headerValue">lcov.info</td>
<td></td>
<td class="headerItem">Lines:</td>
<td class="headerCovTableEntry">684</td>
<td class="headerCovTableEntry">684</td>
<td class="headerCovTableEntryHi">100.0 %</td>
</tr>
<tr>
<td class="headerItem">Date:</td>
<td class="headerValue">2016-11-21</td>
<td></td>
<td class="headerItem">Functions:</td>
<td class="headerCovTableEntry">14</td>
<td class="headerCovTableEntry">14</td>
<td class="headerCovTableEntryHi">100.0 %</td>
</tr>
<tr><td><img src="../glass.png" width=3 height=3 alt=""></td></tr>
</table>
</td>
</tr>
<tr><td class="ruler"><img src="../glass.png" width=3 height=3 alt=""></td></tr>
</table>
<table cellpadding=0 cellspacing=0 border=0>
<tr>
<td><br></td>
</tr>
<tr>
<td>
<pre class="sourceHeading"> Line data Source code</pre>
<pre class="source">
<a name="1"><span class="lineNum"> 1 </span> : /*</a>
<span class="lineNum"> 2 </span> : * Copyright (C) 2013 Andrea Mazzoleni
<span class="lineNum"> 3 </span> : *
<span class="lineNum"> 4 </span> : * This program is free software: you can redistribute it and/or modify
<span class="lineNum"> 5 </span> : * it under the terms of the GNU General Public License as published by
<span class="lineNum"> 6 </span> : * the Free Software Foundation, either version 2 of the License, or
<span class="lineNum"> 7 </span> : * (at your option) any later version.
<span class="lineNum"> 8 </span> : *
<span class="lineNum"> 9 </span> : * This program is distributed in the hope that it will be useful,
<span class="lineNum"> 10 </span> : * but WITHOUT ANY WARRANTY; without even the implied warranty of
<span class="lineNum"> 11 </span> : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
<span class="lineNum"> 12 </span> : * GNU General Public License for more details.
<span class="lineNum"> 13 </span> : */
<span class="lineNum"> 14 </span> :
<span class="lineNum"> 15 </span> : #include &quot;internal.h&quot;
<span class="lineNum"> 16 </span> : #include &quot;gf.h&quot;
<span class="lineNum"> 17 </span> :
<span class="lineNum"> 18 </span> : /*
<span class="lineNum"> 19 </span> : * For x86 optimizations you can see:
<span class="lineNum"> 20 </span> : *
<span class="lineNum"> 21 </span> : * Software optimization resources
<span class="lineNum"> 22 </span> : * http://www.agner.org/optimize/
<span class="lineNum"> 23 </span> : *
<span class="lineNum"> 24 </span> : * x86, x64 Instruction Latency, Memory Latency and CPUID dumps
<span class="lineNum"> 25 </span> : * http://users.atw.hu/instlatx64/
<span class="lineNum"> 26 </span> : */
<span class="lineNum"> 27 </span> :
<span class="lineNum"> 28 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSE2)
<span class="lineNum"> 29 </span> : /*
<span class="lineNum"> 30 </span> : * GEN1 (RAID5 with xor) SSE2 implementation
<span class="lineNum"> 31 </span> : *
<span class="lineNum"> 32 </span> : * Intentionally don't process more than 64 bytes because 64 is the typical
<span class="lineNum"> 33 </span> : * cache block, and processing 128 bytes doesn't increase performance, and in
<a name="34"><span class="lineNum"> 34 </span> : * some cases it even decreases it.</a>
<span class="lineNum"> 35 </span> : */
<span class="lineNum"> 36 </span><span class="lineCov"> 288948 : void raid_gen1_sse2(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 37 </span> : {
<span class="lineNum"> 38 </span><span class="lineCov"> 288948 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 39 </span> : uint8_t *p;
<span class="lineNum"> 40 </span> : int d, l;
<span class="lineNum"> 41 </span> : size_t i;
<span class="lineNum"> 42 </span> :
<span class="lineNum"> 43 </span><span class="lineCov"> 288948 : l = nd - 1;</span>
<span class="lineNum"> 44 </span><span class="lineCov"> 288948 : p = v[nd];</span>
<span class="lineNum"> 45 </span> :
<span class="lineNum"> 46 </span> : raid_sse_begin();
<span class="lineNum"> 47 </span> :
<span class="lineNum"> 48 </span><span class="lineCov"> 6140160 : for (i = 0; i &lt; size; i += 64) {</span>
<span class="lineNum"> 49 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 50 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (v[l][i + 16]));</span>
<span class="lineNum"> 51 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (v[l][i + 32]));</span>
<span class="lineNum"> 52 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (v[l][i + 48]));</span>
<span class="lineNum"> 53 </span><span class="lineCov"> 37573252 : for (d = l - 1; d &gt;= 0; --d) {</span>
<span class="lineNum"> 54 </span><span class="lineCov"> 31722040 : asm volatile (&quot;pxor %0,%%xmm0&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 55 </span><span class="lineCov"> 31722040 : asm volatile (&quot;pxor %0,%%xmm1&quot; : : &quot;m&quot; (v[d][i + 16]));</span>
<span class="lineNum"> 56 </span><span class="lineCov"> 31722040 : asm volatile (&quot;pxor %0,%%xmm2&quot; : : &quot;m&quot; (v[d][i + 32]));</span>
<span class="lineNum"> 57 </span><span class="lineCov"> 31722040 : asm volatile (&quot;pxor %0,%%xmm3&quot; : : &quot;m&quot; (v[d][i + 48]));</span>
<span class="lineNum"> 58 </span> : }
<span class="lineNum"> 59 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 60 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (p[i + 16]));</span>
<span class="lineNum"> 61 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (p[i + 32]));</span>
<span class="lineNum"> 62 </span><span class="lineCov"> 5851212 : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (p[i + 48]));</span>
<span class="lineNum"> 63 </span> : }
<span class="lineNum"> 64 </span> :
<span class="lineNum"> 65 </span> : raid_sse_end();
<span class="lineNum"> 66 </span><span class="lineCov"> 288948 : }</span>
<span class="lineNum"> 67 </span> : #endif
<span class="lineNum"> 68 </span> :
<span class="lineNum"> 69 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 70 </span> : /*
<span class="lineNum"> 71 </span> : * GEN1 (RAID5 with xor) AVX2 implementation
<span class="lineNum"> 72 </span> : *
<span class="lineNum"> 73 </span> : * Intentionally don't process more than 64 bytes because 64 is the typical
<span class="lineNum"> 74 </span> : * cache block, and processing 128 bytes doesn't increase performance, and in
<a name="75"><span class="lineNum"> 75 </span> : * some cases it even decreases it.</a>
<span class="lineNum"> 76 </span> : */
<span class="lineNum"> 77 </span><span class="lineCov"> 97 : void raid_gen1_avx2(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 78 </span> : {
<span class="lineNum"> 79 </span><span class="lineCov"> 97 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 80 </span> : uint8_t *p;
<span class="lineNum"> 81 </span> : int d, l;
<span class="lineNum"> 82 </span> : size_t i;
<span class="lineNum"> 83 </span> :
<span class="lineNum"> 84 </span><span class="lineCov"> 97 : l = nd - 1;</span>
<span class="lineNum"> 85 </span><span class="lineCov"> 97 : p = v[nd];</span>
<span class="lineNum"> 86 </span> :
<span class="lineNum"> 87 </span> : raid_avx_begin();
<span class="lineNum"> 88 </span> :
<span class="lineNum"> 89 </span><span class="lineCov"> 50189 : for (i = 0; i &lt; size; i += 64) {</span>
<span class="lineNum"> 90 </span><span class="lineCov"> 50092 : asm volatile (&quot;vmovdqa %0,%%ymm0&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 91 </span><span class="lineCov"> 50092 : asm volatile (&quot;vmovdqa %0,%%ymm1&quot; : : &quot;m&quot; (v[l][i + 32]));</span>
<span class="lineNum"> 92 </span><span class="lineCov"> 407172 : for (d = l - 1; d &gt;= 0; --d) {</span>
<span class="lineNum"> 93 </span><span class="lineCov"> 357080 : asm volatile (&quot;vpxor %0,%%ymm0,%%ymm0&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 94 </span><span class="lineCov"> 357080 : asm volatile (&quot;vpxor %0,%%ymm1,%%ymm1&quot; : : &quot;m&quot; (v[d][i + 32]));</span>
<span class="lineNum"> 95 </span> : }
<span class="lineNum"> 96 </span><span class="lineCov"> 50092 : asm volatile (&quot;vmovntdq %%ymm0,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 97 </span><span class="lineCov"> 50092 : asm volatile (&quot;vmovntdq %%ymm1,%0&quot; : &quot;=m&quot; (p[i + 32]));</span>
<span class="lineNum"> 98 </span> : }
<span class="lineNum"> 99 </span> :
<span class="lineNum"> 100 </span> : raid_avx_end();
<span class="lineNum"> 101 </span><span class="lineCov"> 97 : }</span>
<span class="lineNum"> 102 </span> : #endif
<span class="lineNum"> 103 </span> :
<span class="lineNum"> 104 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSE2)
<span class="lineNum"> 105 </span> : static const struct gfconst16 {
<span class="lineNum"> 106 </span> : uint8_t poly[16];
<span class="lineNum"> 107 </span> : uint8_t low4[16];
<span class="lineNum"> 108 </span> : } gfconst16 __aligned(32) = {
<span class="lineNum"> 109 </span> : {
<span class="lineNum"> 110 </span> : 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d,
<span class="lineNum"> 111 </span> : 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d
<span class="lineNum"> 112 </span> : },
<span class="lineNum"> 113 </span> : {
<span class="lineNum"> 114 </span> : 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
<span class="lineNum"> 115 </span> : 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f
<span class="lineNum"> 116 </span> : },
<span class="lineNum"> 117 </span> : };
<span class="lineNum"> 118 </span> : #endif
<span class="lineNum"> 119 </span> :
<span class="lineNum"> 120 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSE2)
<span class="lineNum"> 121 </span> : /*
<a name="122"><span class="lineNum"> 122 </span> : * GEN2 (RAID6 with powers of 2) SSE2 implementation</a>
<span class="lineNum"> 123 </span> : */
<span class="lineNum"> 124 </span><span class="lineCov"> 268716 : void raid_gen2_sse2(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 125 </span> : {
<span class="lineNum"> 126 </span><span class="lineCov"> 268716 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 127 </span> : uint8_t *p;
<span class="lineNum"> 128 </span> : uint8_t *q;
<span class="lineNum"> 129 </span> : int d, l;
<span class="lineNum"> 130 </span> : size_t i;
<span class="lineNum"> 131 </span> :
<span class="lineNum"> 132 </span><span class="lineCov"> 268716 : l = nd - 1;</span>
<span class="lineNum"> 133 </span><span class="lineCov"> 268716 : p = v[nd];</span>
<span class="lineNum"> 134 </span><span class="lineCov"> 268716 : q = v[nd + 1];</span>
<span class="lineNum"> 135 </span> :
<span class="lineNum"> 136 </span> : raid_sse_begin();
<span class="lineNum"> 137 </span> :
<span class="lineNum"> 138 </span><span class="lineCov"> 268716 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 139 </span> :
<span class="lineNum"> 140 </span><span class="lineCov"> 10785156 : for (i = 0; i &lt; size; i += 32) {</span>
<span class="lineNum"> 141 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 142 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (v[l][i + 16]));</span>
<span class="lineNum"> 143 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movdqa %xmm0,%xmm2&quot;);</span>
<span class="lineNum"> 144 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movdqa %xmm1,%xmm3&quot;);</span>
<span class="lineNum"> 145 </span><span class="lineCov"> 66949256 : for (d = l - 1; d &gt;= 0; --d) {</span>
<span class="lineNum"> 146 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm4,%xmm4&quot;);</span>
<span class="lineNum"> 147 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 148 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pcmpgtb %xmm2,%xmm4&quot;);</span>
<span class="lineNum"> 149 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pcmpgtb %xmm3,%xmm5&quot;);</span>
<span class="lineNum"> 150 </span><span class="lineCov"> 56432816 : asm volatile (&quot;paddb %xmm2,%xmm2&quot;);</span>
<span class="lineNum"> 151 </span><span class="lineCov"> 56432816 : asm volatile (&quot;paddb %xmm3,%xmm3&quot;);</span>
<span class="lineNum"> 152 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 153 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 154 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 155 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 156 </span> :
<span class="lineNum"> 157 </span><span class="lineCov"> 56432816 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 158 </span><span class="lineCov"> 56432816 : asm volatile (&quot;movdqa %0,%%xmm5&quot; : : &quot;m&quot; (v[d][i + 16]));</span>
<span class="lineNum"> 159 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 160 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);</span>
<span class="lineNum"> 161 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 162 </span><span class="lineCov"> 56432816 : asm volatile (&quot;pxor %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 163 </span> : }
<span class="lineNum"> 164 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 165 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (p[i + 16]));</span>
<span class="lineNum"> 166 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (q[i]));</span>
<span class="lineNum"> 167 </span><span class="lineCov"> 10516440 : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (q[i + 16]));</span>
<span class="lineNum"> 168 </span> : }
<span class="lineNum"> 169 </span> :
<span class="lineNum"> 170 </span> : raid_sse_end();
<span class="lineNum"> 171 </span><span class="lineCov"> 268716 : }</span>
<span class="lineNum"> 172 </span> : #endif
<span class="lineNum"> 173 </span> :
<span class="lineNum"> 174 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 175 </span> : /*
<a name="176"><span class="lineNum"> 176 </span> : * GEN2 (RAID6 with powers of 2) AVX2 implementation</a>
<span class="lineNum"> 177 </span> : */
<span class="lineNum"> 178 </span><span class="lineCov"> 510 : void raid_gen2_avx2(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 179 </span> : {
<span class="lineNum"> 180 </span><span class="lineCov"> 510 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 181 </span> : uint8_t *p;
<span class="lineNum"> 182 </span> : uint8_t *q;
<span class="lineNum"> 183 </span> : int d, l;
<span class="lineNum"> 184 </span> : size_t i;
<span class="lineNum"> 185 </span> :
<span class="lineNum"> 186 </span><span class="lineCov"> 510 : l = nd - 1;</span>
<span class="lineNum"> 187 </span><span class="lineCov"> 510 : p = v[nd];</span>
<span class="lineNum"> 188 </span><span class="lineCov"> 510 : q = v[nd + 1];</span>
<span class="lineNum"> 189 </span> :
<span class="lineNum"> 190 </span> : raid_avx_begin();
<span class="lineNum"> 191 </span> :
<span class="lineNum"> 192 </span><span class="lineCov"> 510 : asm volatile (&quot;vbroadcasti128 %0, %%ymm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 193 </span><span class="lineCov"> 510 : asm volatile (&quot;vpxor %ymm6,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 194 </span> :
<span class="lineNum"> 195 </span><span class="lineCov"> 113754 : for (i = 0; i &lt; size; i += 64) {</span>
<span class="lineNum"> 196 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovdqa %0,%%ymm0&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 197 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovdqa %0,%%ymm1&quot; : : &quot;m&quot; (v[l][i + 32]));</span>
<span class="lineNum"> 198 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovdqa %ymm0,%ymm2&quot;);</span>
<span class="lineNum"> 199 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovdqa %ymm1,%ymm3&quot;);</span>
<span class="lineNum"> 200 </span><span class="lineCov"> 919748 : for (d = l - 1; d &gt;= 0; --d) {</span>
<span class="lineNum"> 201 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpcmpgtb %ymm2,%ymm6,%ymm4&quot;);</span>
<span class="lineNum"> 202 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpcmpgtb %ymm3,%ymm6,%ymm5&quot;);</span>
<span class="lineNum"> 203 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpaddb %ymm2,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 204 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpaddb %ymm3,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 205 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpand %ymm7,%ymm4,%ymm4&quot;);</span>
<span class="lineNum"> 206 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);</span>
<span class="lineNum"> 207 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpxor %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 208 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpxor %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 209 </span> :
<span class="lineNum"> 210 </span><span class="lineCov"> 806504 : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 211 </span><span class="lineCov"> 806504 : asm volatile (&quot;vmovdqa %0,%%ymm5&quot; : : &quot;m&quot; (v[d][i + 32]));</span>
<span class="lineNum"> 212 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpxor %ymm4,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 213 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpxor %ymm5,%ymm1,%ymm1&quot;);</span>
<span class="lineNum"> 214 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpxor %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 215 </span><span class="lineCov"> 806504 : asm volatile (&quot;vpxor %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 216 </span> : }
<span class="lineNum"> 217 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovntdq %%ymm0,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 218 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovntdq %%ymm1,%0&quot; : &quot;=m&quot; (p[i + 32]));</span>
<span class="lineNum"> 219 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovntdq %%ymm2,%0&quot; : &quot;=m&quot; (q[i]));</span>
<span class="lineNum"> 220 </span><span class="lineCov"> 113244 : asm volatile (&quot;vmovntdq %%ymm3,%0&quot; : &quot;=m&quot; (q[i + 32]));</span>
<span class="lineNum"> 221 </span> : }
<span class="lineNum"> 222 </span> :
<span class="lineNum"> 223 </span> : raid_avx_end();
<span class="lineNum"> 224 </span><span class="lineCov"> 510 : }</span>
<span class="lineNum"> 225 </span> : #endif
<span class="lineNum"> 226 </span> :
<span class="lineNum"> 227 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_SSE2)
<span class="lineNum"> 228 </span> : /*
<span class="lineNum"> 229 </span> : * GEN2 (RAID6 with powers of 2) SSE2 implementation
<span class="lineNum"> 230 </span> : *
<span class="lineNum"> 231 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 232 </span> : */
<span class="lineNum"> 233 </span> : void raid_gen2_sse2ext(int nd, size_t size, void **vv)
<span class="lineNum"> 234 </span> : {
<span class="lineNum"> 235 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 236 </span> : uint8_t *p;
<span class="lineNum"> 237 </span> : uint8_t *q;
<span class="lineNum"> 238 </span> : int d, l;
<span class="lineNum"> 239 </span> : size_t i;
<span class="lineNum"> 240 </span> :
<span class="lineNum"> 241 </span> : l = nd - 1;
<span class="lineNum"> 242 </span> : p = v[nd];
<span class="lineNum"> 243 </span> : q = v[nd + 1];
<span class="lineNum"> 244 </span> :
<span class="lineNum"> 245 </span> : raid_sse_begin();
<span class="lineNum"> 246 </span> :
<span class="lineNum"> 247 </span> : asm volatile (&quot;movdqa %0,%%xmm15&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 248 </span> :
<span class="lineNum"> 249 </span> : for (i = 0; i &lt; size; i += 64) {
<span class="lineNum"> 250 </span> : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 251 </span> : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (v[l][i + 16]));
<span class="lineNum"> 252 </span> : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (v[l][i + 32]));
<span class="lineNum"> 253 </span> : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (v[l][i + 48]));
<span class="lineNum"> 254 </span> : asm volatile (&quot;movdqa %xmm0,%xmm4&quot;);
<span class="lineNum"> 255 </span> : asm volatile (&quot;movdqa %xmm1,%xmm5&quot;);
<span class="lineNum"> 256 </span> : asm volatile (&quot;movdqa %xmm2,%xmm6&quot;);
<span class="lineNum"> 257 </span> : asm volatile (&quot;movdqa %xmm3,%xmm7&quot;);
<span class="lineNum"> 258 </span> : for (d = l - 1; d &gt;= 0; --d) {
<span class="lineNum"> 259 </span> : asm volatile (&quot;pxor %xmm8,%xmm8&quot;);
<span class="lineNum"> 260 </span> : asm volatile (&quot;pxor %xmm9,%xmm9&quot;);
<span class="lineNum"> 261 </span> : asm volatile (&quot;pxor %xmm10,%xmm10&quot;);
<span class="lineNum"> 262 </span> : asm volatile (&quot;pxor %xmm11,%xmm11&quot;);
<span class="lineNum"> 263 </span> : asm volatile (&quot;pcmpgtb %xmm4,%xmm8&quot;);
<span class="lineNum"> 264 </span> : asm volatile (&quot;pcmpgtb %xmm5,%xmm9&quot;);
<span class="lineNum"> 265 </span> : asm volatile (&quot;pcmpgtb %xmm6,%xmm10&quot;);
<span class="lineNum"> 266 </span> : asm volatile (&quot;pcmpgtb %xmm7,%xmm11&quot;);
<span class="lineNum"> 267 </span> : asm volatile (&quot;paddb %xmm4,%xmm4&quot;);
<span class="lineNum"> 268 </span> : asm volatile (&quot;paddb %xmm5,%xmm5&quot;);
<span class="lineNum"> 269 </span> : asm volatile (&quot;paddb %xmm6,%xmm6&quot;);
<span class="lineNum"> 270 </span> : asm volatile (&quot;paddb %xmm7,%xmm7&quot;);
<span class="lineNum"> 271 </span> : asm volatile (&quot;pand %xmm15,%xmm8&quot;);
<span class="lineNum"> 272 </span> : asm volatile (&quot;pand %xmm15,%xmm9&quot;);
<span class="lineNum"> 273 </span> : asm volatile (&quot;pand %xmm15,%xmm10&quot;);
<span class="lineNum"> 274 </span> : asm volatile (&quot;pand %xmm15,%xmm11&quot;);
<span class="lineNum"> 275 </span> : asm volatile (&quot;pxor %xmm8,%xmm4&quot;);
<span class="lineNum"> 276 </span> : asm volatile (&quot;pxor %xmm9,%xmm5&quot;);
<span class="lineNum"> 277 </span> : asm volatile (&quot;pxor %xmm10,%xmm6&quot;);
<span class="lineNum"> 278 </span> : asm volatile (&quot;pxor %xmm11,%xmm7&quot;);
<span class="lineNum"> 279 </span> :
<span class="lineNum"> 280 </span> : asm volatile (&quot;movdqa %0,%%xmm8&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 281 </span> : asm volatile (&quot;movdqa %0,%%xmm9&quot; : : &quot;m&quot; (v[d][i + 16]));
<span class="lineNum"> 282 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[d][i + 32]));
<span class="lineNum"> 283 </span> : asm volatile (&quot;movdqa %0,%%xmm11&quot; : : &quot;m&quot; (v[d][i + 48]));
<span class="lineNum"> 284 </span> : asm volatile (&quot;pxor %xmm8,%xmm0&quot;);
<span class="lineNum"> 285 </span> : asm volatile (&quot;pxor %xmm9,%xmm1&quot;);
<span class="lineNum"> 286 </span> : asm volatile (&quot;pxor %xmm10,%xmm2&quot;);
<span class="lineNum"> 287 </span> : asm volatile (&quot;pxor %xmm11,%xmm3&quot;);
<span class="lineNum"> 288 </span> : asm volatile (&quot;pxor %xmm8,%xmm4&quot;);
<span class="lineNum"> 289 </span> : asm volatile (&quot;pxor %xmm9,%xmm5&quot;);
<span class="lineNum"> 290 </span> : asm volatile (&quot;pxor %xmm10,%xmm6&quot;);
<span class="lineNum"> 291 </span> : asm volatile (&quot;pxor %xmm11,%xmm7&quot;);
<span class="lineNum"> 292 </span> : }
<span class="lineNum"> 293 </span> : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 294 </span> : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (p[i + 16]));
<span class="lineNum"> 295 </span> : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (p[i + 32]));
<span class="lineNum"> 296 </span> : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (p[i + 48]));
<span class="lineNum"> 297 </span> : asm volatile (&quot;movntdq %%xmm4,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 298 </span> : asm volatile (&quot;movntdq %%xmm5,%0&quot; : &quot;=m&quot; (q[i + 16]));
<span class="lineNum"> 299 </span> : asm volatile (&quot;movntdq %%xmm6,%0&quot; : &quot;=m&quot; (q[i + 32]));
<span class="lineNum"> 300 </span> : asm volatile (&quot;movntdq %%xmm7,%0&quot; : &quot;=m&quot; (q[i + 48]));
<span class="lineNum"> 301 </span> : }
<span class="lineNum"> 302 </span> :
<span class="lineNum"> 303 </span> : raid_sse_end();
<span class="lineNum"> 304 </span> : }
<span class="lineNum"> 305 </span> : #endif
<span class="lineNum"> 306 </span> :
<span class="lineNum"> 307 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 308 </span> : /*
<a name="309"><span class="lineNum"> 309 </span> : * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation</a>
<span class="lineNum"> 310 </span> : */
<span class="lineNum"> 311 </span><span class="lineCov"> 29542 : void raid_gen3_ssse3(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 312 </span> : {
<span class="lineNum"> 313 </span><span class="lineCov"> 29542 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 314 </span> : uint8_t *p;
<span class="lineNum"> 315 </span> : uint8_t *q;
<span class="lineNum"> 316 </span> : uint8_t *r;
<span class="lineNum"> 317 </span> : int d, l;
<span class="lineNum"> 318 </span> : size_t i;
<span class="lineNum"> 319 </span> :
<span class="lineNum"> 320 </span><span class="lineCov"> 29542 : l = nd - 1;</span>
<span class="lineNum"> 321 </span><span class="lineCov"> 29542 : p = v[nd];</span>
<span class="lineNum"> 322 </span><span class="lineCov"> 29542 : q = v[nd + 1];</span>
<span class="lineNum"> 323 </span><span class="lineCov"> 29542 : r = v[nd + 2];</span>
<span class="lineNum"> 324 </span> :
<span class="lineNum"> 325 </span> : /* special case with only one data disk */
<span class="lineNum"> 326 </span><span class="lineCov"> 29542 : if (l == 0) {</span>
<span class="lineNum"> 327 </span><span class="lineCov"> 4 : for (i = 0; i &lt; 3; ++i)</span>
<span class="lineNum"> 328 </span><span class="lineCov"> 3 : memcpy(v[1 + i], v[0], size);</span>
<span class="lineNum"> 329 </span><span class="lineCov"> 29543 : return;</span>
<span class="lineNum"> 330 </span> : }
<span class="lineNum"> 331 </span> :
<span class="lineNum"> 332 </span> : raid_sse_begin();
<span class="lineNum"> 333 </span> :
<span class="lineNum"> 334 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 335 </span><span class="lineCov"> 29541 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 336 </span><span class="lineCov"> 29541 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 337 </span> :
<span class="lineNum"> 338 </span><span class="lineCov"> 4888821 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 339 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 340 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 341 </span> :
<span class="lineNum"> 342 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 343 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 344 </span> :
<span class="lineNum"> 345 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 346 </span><span class="lineCov"> 4859280 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 347 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 348 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 349 </span> :
<span class="lineNum"> 350 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));</span>
<span class="lineNum"> 351 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));</span>
<span class="lineNum"> 352 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 353 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pshufb %xmm5,%xmm6&quot;);</span>
<span class="lineNum"> 354 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);</span>
<span class="lineNum"> 355 </span> :
<span class="lineNum"> 356 </span> : /* intermediate disks */
<span class="lineNum"> 357 </span><span class="lineCov"> 30489328 : for (d = l - 1; d &gt; 0; --d) {</span>
<span class="lineNum"> 358 </span><span class="lineCov"> 25630048 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 359 </span> :
<span class="lineNum"> 360 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 361 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);</span>
<span class="lineNum"> 362 </span><span class="lineCov"> 25630048 : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);</span>
<span class="lineNum"> 363 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pand %xmm3,%xmm5&quot;);</span>
<span class="lineNum"> 364 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);</span>
<span class="lineNum"> 365 </span> :
<span class="lineNum"> 366 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 367 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 368 </span> :
<span class="lineNum"> 369 </span><span class="lineCov"> 25630048 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 370 </span><span class="lineCov"> 25630048 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 371 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 372 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 373 </span> :
<span class="lineNum"> 374 </span><span class="lineCov"> 25630048 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));</span>
<span class="lineNum"> 375 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 376 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);</span>
<span class="lineNum"> 377 </span><span class="lineCov"> 25630048 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));</span>
<span class="lineNum"> 378 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pshufb %xmm5,%xmm6&quot;);</span>
<span class="lineNum"> 379 </span><span class="lineCov"> 25630048 : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);</span>
<span class="lineNum"> 380 </span> : }
<span class="lineNum"> 381 </span> :
<span class="lineNum"> 382 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 383 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[0][i]));</span>
<span class="lineNum"> 384 </span> :
<span class="lineNum"> 385 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 386 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);</span>
<span class="lineNum"> 387 </span><span class="lineCov"> 4859280 : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);</span>
<span class="lineNum"> 388 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pand %xmm3,%xmm5&quot;);</span>
<span class="lineNum"> 389 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);</span>
<span class="lineNum"> 390 </span> :
<span class="lineNum"> 391 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 392 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 393 </span><span class="lineCov"> 4859280 : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 394 </span> :
<span class="lineNum"> 395 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 396 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (q[i]));</span>
<span class="lineNum"> 397 </span><span class="lineCov"> 4859280 : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (r[i]));</span>
<span class="lineNum"> 398 </span> : }
<span class="lineNum"> 399 </span> :
<span class="lineNum"> 400 </span> : raid_sse_end();
<span class="lineNum"> 401 </span> : }
<span class="lineNum"> 402 </span> : #endif
<span class="lineNum"> 403 </span> :
<span class="lineNum"> 404 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 405 </span> : /*
<span class="lineNum"> 406 </span> : * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
<span class="lineNum"> 407 </span> : *
<span class="lineNum"> 408 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 409 </span> : */
<span class="lineNum"> 410 </span> : void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
<span class="lineNum"> 411 </span> : {
<span class="lineNum"> 412 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 413 </span> : uint8_t *p;
<span class="lineNum"> 414 </span> : uint8_t *q;
<span class="lineNum"> 415 </span> : uint8_t *r;
<span class="lineNum"> 416 </span> : int d, l;
<span class="lineNum"> 417 </span> : size_t i;
<span class="lineNum"> 418 </span> :
<span class="lineNum"> 419 </span> : l = nd - 1;
<span class="lineNum"> 420 </span> : p = v[nd];
<span class="lineNum"> 421 </span> : q = v[nd + 1];
<span class="lineNum"> 422 </span> : r = v[nd + 2];
<span class="lineNum"> 423 </span> :
<span class="lineNum"> 424 </span> : /* special case with only one data disk */
<span class="lineNum"> 425 </span> : if (l == 0) {
<span class="lineNum"> 426 </span> : for (i = 0; i &lt; 3; ++i)
<span class="lineNum"> 427 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 428 </span> : return;
<span class="lineNum"> 429 </span> : }
<span class="lineNum"> 430 </span> :
<span class="lineNum"> 431 </span> : raid_sse_begin();
<span class="lineNum"> 432 </span> :
<span class="lineNum"> 433 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 434 </span> : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 435 </span> : asm volatile (&quot;movdqa %0,%%xmm11&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 436 </span> :
<span class="lineNum"> 437 </span> : for (i = 0; i &lt; size; i += 32) {
<span class="lineNum"> 438 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 439 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 440 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (v[l][i + 16]));
<span class="lineNum"> 441 </span> :
<span class="lineNum"> 442 </span> : asm volatile (&quot;movdqa %xmm4,%xmm0&quot;);
<span class="lineNum"> 443 </span> : asm volatile (&quot;movdqa %xmm4,%xmm1&quot;);
<span class="lineNum"> 444 </span> : asm volatile (&quot;movdqa %xmm12,%xmm8&quot;);
<span class="lineNum"> 445 </span> : asm volatile (&quot;movdqa %xmm12,%xmm9&quot;);
<span class="lineNum"> 446 </span> :
<span class="lineNum"> 447 </span> : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);
<span class="lineNum"> 448 </span> : asm volatile (&quot;movdqa %xmm12,%xmm13&quot;);
<span class="lineNum"> 449 </span> : asm volatile (&quot;psrlw $4,%xmm5&quot;);
<span class="lineNum"> 450 </span> : asm volatile (&quot;psrlw $4,%xmm13&quot;);
<span class="lineNum"> 451 </span> : asm volatile (&quot;pand %xmm11,%xmm4&quot;);
<span class="lineNum"> 452 </span> : asm volatile (&quot;pand %xmm11,%xmm12&quot;);
<span class="lineNum"> 453 </span> : asm volatile (&quot;pand %xmm11,%xmm5&quot;);
<span class="lineNum"> 454 </span> : asm volatile (&quot;pand %xmm11,%xmm13&quot;);
<span class="lineNum"> 455 </span> :
<span class="lineNum"> 456 </span> : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 457 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 458 </span> : asm volatile (&quot;movdqa %xmm2,%xmm10&quot;);
<span class="lineNum"> 459 </span> : asm volatile (&quot;movdqa %xmm7,%xmm15&quot;);
<span class="lineNum"> 460 </span> : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);
<span class="lineNum"> 461 </span> : asm volatile (&quot;pshufb %xmm12,%xmm10&quot;);
<span class="lineNum"> 462 </span> : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);
<span class="lineNum"> 463 </span> : asm volatile (&quot;pshufb %xmm13,%xmm15&quot;);
<span class="lineNum"> 464 </span> : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);
<span class="lineNum"> 465 </span> : asm volatile (&quot;pxor %xmm15,%xmm10&quot;);
<span class="lineNum"> 466 </span> :
<span class="lineNum"> 467 </span> : /* intermediate disks */
<span class="lineNum"> 468 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 469 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 470 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (v[d][i + 16]));
<span class="lineNum"> 471 </span> :
<span class="lineNum"> 472 </span> : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);
<span class="lineNum"> 473 </span> : asm volatile (&quot;pxor %xmm13,%xmm13&quot;);
<span class="lineNum"> 474 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);
<span class="lineNum"> 475 </span> : asm volatile (&quot;pcmpgtb %xmm9,%xmm13&quot;);
<span class="lineNum"> 476 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 477 </span> : asm volatile (&quot;paddb %xmm9,%xmm9&quot;);
<span class="lineNum"> 478 </span> : asm volatile (&quot;pand %xmm3,%xmm5&quot;);
<span class="lineNum"> 479 </span> : asm volatile (&quot;pand %xmm3,%xmm13&quot;);
<span class="lineNum"> 480 </span> : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);
<span class="lineNum"> 481 </span> : asm volatile (&quot;pxor %xmm13,%xmm9&quot;);
<span class="lineNum"> 482 </span> :
<span class="lineNum"> 483 </span> : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);
<span class="lineNum"> 484 </span> : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);
<span class="lineNum"> 485 </span> : asm volatile (&quot;pxor %xmm12,%xmm8&quot;);
<span class="lineNum"> 486 </span> : asm volatile (&quot;pxor %xmm12,%xmm9&quot;);
<span class="lineNum"> 487 </span> :
<span class="lineNum"> 488 </span> : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);
<span class="lineNum"> 489 </span> : asm volatile (&quot;movdqa %xmm12,%xmm13&quot;);
<span class="lineNum"> 490 </span> : asm volatile (&quot;psrlw $4,%xmm5&quot;);
<span class="lineNum"> 491 </span> : asm volatile (&quot;psrlw $4,%xmm13&quot;);
<span class="lineNum"> 492 </span> : asm volatile (&quot;pand %xmm11,%xmm4&quot;);
<span class="lineNum"> 493 </span> : asm volatile (&quot;pand %xmm11,%xmm12&quot;);
<span class="lineNum"> 494 </span> : asm volatile (&quot;pand %xmm11,%xmm5&quot;);
<span class="lineNum"> 495 </span> : asm volatile (&quot;pand %xmm11,%xmm13&quot;);
<span class="lineNum"> 496 </span> :
<span class="lineNum"> 497 </span> : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 498 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 499 </span> : asm volatile (&quot;movdqa %xmm6,%xmm14&quot;);
<span class="lineNum"> 500 </span> : asm volatile (&quot;movdqa %xmm7,%xmm15&quot;);
<span class="lineNum"> 501 </span> : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);
<span class="lineNum"> 502 </span> : asm volatile (&quot;pshufb %xmm12,%xmm14&quot;);
<span class="lineNum"> 503 </span> : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);
<span class="lineNum"> 504 </span> : asm volatile (&quot;pshufb %xmm13,%xmm15&quot;);
<span class="lineNum"> 505 </span> : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);
<span class="lineNum"> 506 </span> : asm volatile (&quot;pxor %xmm14,%xmm10&quot;);
<span class="lineNum"> 507 </span> : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);
<span class="lineNum"> 508 </span> : asm volatile (&quot;pxor %xmm15,%xmm10&quot;);
<span class="lineNum"> 509 </span> : }
<span class="lineNum"> 510 </span> :
<span class="lineNum"> 511 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 512 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 513 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (v[0][i + 16]));
<span class="lineNum"> 514 </span> :
<span class="lineNum"> 515 </span> : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);
<span class="lineNum"> 516 </span> : asm volatile (&quot;pxor %xmm13,%xmm13&quot;);
<span class="lineNum"> 517 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);
<span class="lineNum"> 518 </span> : asm volatile (&quot;pcmpgtb %xmm9,%xmm13&quot;);
<span class="lineNum"> 519 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 520 </span> : asm volatile (&quot;paddb %xmm9,%xmm9&quot;);
<span class="lineNum"> 521 </span> : asm volatile (&quot;pand %xmm3,%xmm5&quot;);
<span class="lineNum"> 522 </span> : asm volatile (&quot;pand %xmm3,%xmm13&quot;);
<span class="lineNum"> 523 </span> : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);
<span class="lineNum"> 524 </span> : asm volatile (&quot;pxor %xmm13,%xmm9&quot;);
<span class="lineNum"> 525 </span> :
<span class="lineNum"> 526 </span> : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);
<span class="lineNum"> 527 </span> : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);
<span class="lineNum"> 528 </span> : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);
<span class="lineNum"> 529 </span> : asm volatile (&quot;pxor %xmm12,%xmm8&quot;);
<span class="lineNum"> 530 </span> : asm volatile (&quot;pxor %xmm12,%xmm9&quot;);
<span class="lineNum"> 531 </span> : asm volatile (&quot;pxor %xmm12,%xmm10&quot;);
<span class="lineNum"> 532 </span> :
<span class="lineNum"> 533 </span> : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 534 </span> : asm volatile (&quot;movntdq %%xmm8,%0&quot; : &quot;=m&quot; (p[i + 16]));
<span class="lineNum"> 535 </span> : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 536 </span> : asm volatile (&quot;movntdq %%xmm9,%0&quot; : &quot;=m&quot; (q[i + 16]));
<span class="lineNum"> 537 </span> : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 538 </span> : asm volatile (&quot;movntdq %%xmm10,%0&quot; : &quot;=m&quot; (r[i + 16]));
<span class="lineNum"> 539 </span> : }
<span class="lineNum"> 540 </span> :
<span class="lineNum"> 541 </span> : raid_sse_end();
<span class="lineNum"> 542 </span> : }
<span class="lineNum"> 543 </span> : #endif
<span class="lineNum"> 544 </span> :
<span class="lineNum"> 545 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 546 </span> : /*
<span class="lineNum"> 547 </span> : * GEN3 (triple parity with Cauchy matrix) AVX2 implementation
<span class="lineNum"> 548 </span> : *
<span class="lineNum"> 549 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 550 </span> : */
<span class="lineNum"> 551 </span> : void raid_gen3_avx2ext(int nd, size_t size, void **vv)
<span class="lineNum"> 552 </span> : {
<span class="lineNum"> 553 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 554 </span> : uint8_t *p;
<span class="lineNum"> 555 </span> : uint8_t *q;
<span class="lineNum"> 556 </span> : uint8_t *r;
<span class="lineNum"> 557 </span> : int d, l;
<span class="lineNum"> 558 </span> : size_t i;
<span class="lineNum"> 559 </span> :
<span class="lineNum"> 560 </span> : l = nd - 1;
<span class="lineNum"> 561 </span> : p = v[nd];
<span class="lineNum"> 562 </span> : q = v[nd + 1];
<span class="lineNum"> 563 </span> : r = v[nd + 2];
<span class="lineNum"> 564 </span> :
<span class="lineNum"> 565 </span> : /* special case with only one data disk */
<span class="lineNum"> 566 </span> : if (l == 0) {
<span class="lineNum"> 567 </span> : for (i = 0; i &lt; 3; ++i)
<span class="lineNum"> 568 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 569 </span> : return;
<span class="lineNum"> 570 </span> : }
<span class="lineNum"> 571 </span> :
<span class="lineNum"> 572 </span> : raid_avx_begin();
<span class="lineNum"> 573 </span> :
<span class="lineNum"> 574 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 575 </span> : asm volatile (&quot;vbroadcasti128 %0, %%ymm3&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 576 </span> : asm volatile (&quot;vbroadcasti128 %0, %%ymm11&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 577 </span> :
<span class="lineNum"> 578 </span> : for (i = 0; i &lt; size; i += 64) {
<span class="lineNum"> 579 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 580 </span> : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 581 </span> : asm volatile (&quot;vmovdqa %0,%%ymm12&quot; : : &quot;m&quot; (v[l][i + 32]));
<span class="lineNum"> 582 </span> :
<span class="lineNum"> 583 </span> : asm volatile (&quot;vmovdqa %ymm4,%ymm0&quot;);
<span class="lineNum"> 584 </span> : asm volatile (&quot;vmovdqa %ymm4,%ymm1&quot;);
<span class="lineNum"> 585 </span> : asm volatile (&quot;vmovdqa %ymm12,%ymm8&quot;);
<span class="lineNum"> 586 </span> : asm volatile (&quot;vmovdqa %ymm12,%ymm9&quot;);
<span class="lineNum"> 587 </span> :
<span class="lineNum"> 588 </span> : asm volatile (&quot;vpsrlw $4,%ymm4,%ymm5&quot;);
<span class="lineNum"> 589 </span> : asm volatile (&quot;vpsrlw $4,%ymm12,%ymm13&quot;);
<span class="lineNum"> 590 </span> : asm volatile (&quot;vpand %ymm11,%ymm4,%ymm4&quot;);
<span class="lineNum"> 591 </span> : asm volatile (&quot;vpand %ymm11,%ymm12,%ymm12&quot;);
<span class="lineNum"> 592 </span> : asm volatile (&quot;vpand %ymm11,%ymm5,%ymm5&quot;);
<span class="lineNum"> 593 </span> : asm volatile (&quot;vpand %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 594 </span> :
<span class="lineNum"> 595 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm10&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 596 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 597 </span> : asm volatile (&quot;vpshufb %ymm4,%ymm10,%ymm2&quot;);
<span class="lineNum"> 598 </span> : asm volatile (&quot;vpshufb %ymm12,%ymm10,%ymm10&quot;);
<span class="lineNum"> 599 </span> : asm volatile (&quot;vpshufb %ymm5,%ymm15,%ymm7&quot;);
<span class="lineNum"> 600 </span> : asm volatile (&quot;vpshufb %ymm13,%ymm15,%ymm15&quot;);
<span class="lineNum"> 601 </span> : asm volatile (&quot;vpxor %ymm7,%ymm2,%ymm2&quot;);
<span class="lineNum"> 602 </span> : asm volatile (&quot;vpxor %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 603 </span> :
<span class="lineNum"> 604 </span> : /* intermediate disks */
<span class="lineNum"> 605 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 606 </span> : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 607 </span> : asm volatile (&quot;vmovdqa %0,%%ymm12&quot; : : &quot;m&quot; (v[d][i + 32]));
<span class="lineNum"> 608 </span> :
<span class="lineNum"> 609 </span> : asm volatile (&quot;vpxor %ymm5,%ymm5,%ymm5&quot;);
<span class="lineNum"> 610 </span> : asm volatile (&quot;vpxor %ymm13,%ymm13,%ymm13&quot;);
<span class="lineNum"> 611 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm5,%ymm5&quot;);
<span class="lineNum"> 612 </span> : asm volatile (&quot;vpcmpgtb %ymm9,%ymm13,%ymm13&quot;);
<span class="lineNum"> 613 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 614 </span> : asm volatile (&quot;vpaddb %ymm9,%ymm9,%ymm9&quot;);
<span class="lineNum"> 615 </span> : asm volatile (&quot;vpand %ymm3,%ymm5,%ymm5&quot;);
<span class="lineNum"> 616 </span> : asm volatile (&quot;vpand %ymm3,%ymm13,%ymm13&quot;);
<span class="lineNum"> 617 </span> : asm volatile (&quot;vpxor %ymm5,%ymm1,%ymm1&quot;);
<span class="lineNum"> 618 </span> : asm volatile (&quot;vpxor %ymm13,%ymm9,%ymm9&quot;);
<span class="lineNum"> 619 </span> :
<span class="lineNum"> 620 </span> : asm volatile (&quot;vpxor %ymm4,%ymm0,%ymm0&quot;);
<span class="lineNum"> 621 </span> : asm volatile (&quot;vpxor %ymm4,%ymm1,%ymm1&quot;);
<span class="lineNum"> 622 </span> : asm volatile (&quot;vpxor %ymm12,%ymm8,%ymm8&quot;);
<span class="lineNum"> 623 </span> : asm volatile (&quot;vpxor %ymm12,%ymm9,%ymm9&quot;);
<span class="lineNum"> 624 </span> :
<span class="lineNum"> 625 </span> : asm volatile (&quot;vpsrlw $4,%ymm4,%ymm5&quot;);
<span class="lineNum"> 626 </span> : asm volatile (&quot;vpsrlw $4,%ymm12,%ymm13&quot;);
<span class="lineNum"> 627 </span> : asm volatile (&quot;vpand %ymm11,%ymm4,%ymm4&quot;);
<span class="lineNum"> 628 </span> : asm volatile (&quot;vpand %ymm11,%ymm12,%ymm12&quot;);
<span class="lineNum"> 629 </span> : asm volatile (&quot;vpand %ymm11,%ymm5,%ymm5&quot;);
<span class="lineNum"> 630 </span> : asm volatile (&quot;vpand %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 631 </span> :
<span class="lineNum"> 632 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm14&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 633 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 634 </span> : asm volatile (&quot;vpshufb %ymm4,%ymm14,%ymm6&quot;);
<span class="lineNum"> 635 </span> : asm volatile (&quot;vpshufb %ymm12,%ymm14,%ymm14&quot;);
<span class="lineNum"> 636 </span> : asm volatile (&quot;vpshufb %ymm5,%ymm15,%ymm7&quot;);
<span class="lineNum"> 637 </span> : asm volatile (&quot;vpshufb %ymm13,%ymm15,%ymm15&quot;);
<span class="lineNum"> 638 </span> : asm volatile (&quot;vpxor %ymm6,%ymm2,%ymm2&quot;);
<span class="lineNum"> 639 </span> : asm volatile (&quot;vpxor %ymm14,%ymm10,%ymm10&quot;);
<span class="lineNum"> 640 </span> : asm volatile (&quot;vpxor %ymm7,%ymm2,%ymm2&quot;);
<span class="lineNum"> 641 </span> : asm volatile (&quot;vpxor %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 642 </span> : }
<span class="lineNum"> 643 </span> :
<span class="lineNum"> 644 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 645 </span> : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 646 </span> : asm volatile (&quot;vmovdqa %0,%%ymm12&quot; : : &quot;m&quot; (v[0][i + 32]));
<span class="lineNum"> 647 </span> :
<span class="lineNum"> 648 </span> : asm volatile (&quot;vpxor %ymm5,%ymm5,%ymm5&quot;);
<span class="lineNum"> 649 </span> : asm volatile (&quot;vpxor %ymm13,%ymm13,%ymm13&quot;);
<span class="lineNum"> 650 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm5,%ymm5&quot;);
<span class="lineNum"> 651 </span> : asm volatile (&quot;vpcmpgtb %ymm9,%ymm13,%ymm13&quot;);
<span class="lineNum"> 652 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 653 </span> : asm volatile (&quot;vpaddb %ymm9,%ymm9,%ymm9&quot;);
<span class="lineNum"> 654 </span> : asm volatile (&quot;vpand %ymm3,%ymm5,%ymm5&quot;);
<span class="lineNum"> 655 </span> : asm volatile (&quot;vpand %ymm3,%ymm13,%ymm13&quot;);
<span class="lineNum"> 656 </span> : asm volatile (&quot;vpxor %ymm5,%ymm1,%ymm1&quot;);
<span class="lineNum"> 657 </span> : asm volatile (&quot;vpxor %ymm13,%ymm9,%ymm9&quot;);
<span class="lineNum"> 658 </span> :
<span class="lineNum"> 659 </span> : asm volatile (&quot;vpxor %ymm4,%ymm0,%ymm0&quot;);
<span class="lineNum"> 660 </span> : asm volatile (&quot;vpxor %ymm4,%ymm1,%ymm1&quot;);
<span class="lineNum"> 661 </span> : asm volatile (&quot;vpxor %ymm4,%ymm2,%ymm2&quot;);
<span class="lineNum"> 662 </span> : asm volatile (&quot;vpxor %ymm12,%ymm8,%ymm8&quot;);
<span class="lineNum"> 663 </span> : asm volatile (&quot;vpxor %ymm12,%ymm9,%ymm9&quot;);
<span class="lineNum"> 664 </span> : asm volatile (&quot;vpxor %ymm12,%ymm10,%ymm10&quot;);
<span class="lineNum"> 665 </span> :
<span class="lineNum"> 666 </span> : asm volatile (&quot;vmovntdq %%ymm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 667 </span> : asm volatile (&quot;vmovntdq %%ymm8,%0&quot; : &quot;=m&quot; (p[i + 32]));
<span class="lineNum"> 668 </span> : asm volatile (&quot;vmovntdq %%ymm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 669 </span> : asm volatile (&quot;vmovntdq %%ymm9,%0&quot; : &quot;=m&quot; (q[i + 32]));
<span class="lineNum"> 670 </span> : asm volatile (&quot;vmovntdq %%ymm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 671 </span> : asm volatile (&quot;vmovntdq %%ymm10,%0&quot; : &quot;=m&quot; (r[i + 32]));
<span class="lineNum"> 672 </span> : }
<span class="lineNum"> 673 </span> :
<span class="lineNum"> 674 </span> : raid_avx_end();
<span class="lineNum"> 675 </span> : }
<span class="lineNum"> 676 </span> : #endif
<span class="lineNum"> 677 </span> :
<span class="lineNum"> 678 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 679 </span> : /*
<a name="680"><span class="lineNum"> 680 </span> : * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation</a>
<span class="lineNum"> 681 </span> : */
<span class="lineNum"> 682 </span><span class="lineCov"> 32469 : void raid_gen4_ssse3(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 683 </span> : {
<span class="lineNum"> 684 </span><span class="lineCov"> 32469 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 685 </span> : uint8_t *p;
<span class="lineNum"> 686 </span> : uint8_t *q;
<span class="lineNum"> 687 </span> : uint8_t *r;
<span class="lineNum"> 688 </span> : uint8_t *s;
<span class="lineNum"> 689 </span> : int d, l;
<span class="lineNum"> 690 </span> : size_t i;
<span class="lineNum"> 691 </span> :
<span class="lineNum"> 692 </span><span class="lineCov"> 32469 : l = nd - 1;</span>
<span class="lineNum"> 693 </span><span class="lineCov"> 32469 : p = v[nd];</span>
<span class="lineNum"> 694 </span><span class="lineCov"> 32469 : q = v[nd + 1];</span>
<span class="lineNum"> 695 </span><span class="lineCov"> 32469 : r = v[nd + 2];</span>
<span class="lineNum"> 696 </span><span class="lineCov"> 32469 : s = v[nd + 3];</span>
<span class="lineNum"> 697 </span> :
<span class="lineNum"> 698 </span> : /* special case with only one data disk */
<span class="lineNum"> 699 </span><span class="lineCov"> 32469 : if (l == 0) {</span>
<span class="lineNum"> 700 </span><span class="lineCov"> 5 : for (i = 0; i &lt; 4; ++i)</span>
<span class="lineNum"> 701 </span><span class="lineCov"> 4 : memcpy(v[1 + i], v[0], size);</span>
<span class="lineNum"> 702 </span><span class="lineCov"> 32470 : return;</span>
<span class="lineNum"> 703 </span> : }
<span class="lineNum"> 704 </span> :
<span class="lineNum"> 705 </span> : raid_sse_begin();
<span class="lineNum"> 706 </span> :
<span class="lineNum"> 707 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 708 </span><span class="lineCov"> 3334804 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 709 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 710 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 711 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 712 </span> :
<span class="lineNum"> 713 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 714 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 715 </span> :
<span class="lineNum"> 716 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 717 </span><span class="lineCov"> 3302336 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 718 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 719 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 720 </span> :
<span class="lineNum"> 721 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));</span>
<span class="lineNum"> 722 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));</span>
<span class="lineNum"> 723 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 724 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 725 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);</span>
<span class="lineNum"> 726 </span> :
<span class="lineNum"> 727 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));</span>
<span class="lineNum"> 728 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));</span>
<span class="lineNum"> 729 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pshufb %xmm4,%xmm3&quot;);</span>
<span class="lineNum"> 730 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 731 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);</span>
<span class="lineNum"> 732 </span> :
<span class="lineNum"> 733 </span> : /* intermediate disks */
<span class="lineNum"> 734 </span><span class="lineCov"> 19771392 : for (d = l - 1; d &gt; 0; --d) {</span>
<span class="lineNum"> 735 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 736 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 737 </span> :
<span class="lineNum"> 738 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 739 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);</span>
<span class="lineNum"> 740 </span><span class="lineCov"> 16469056 : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);</span>
<span class="lineNum"> 741 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 742 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);</span>
<span class="lineNum"> 743 </span> :
<span class="lineNum"> 744 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 745 </span> :
<span class="lineNum"> 746 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 747 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 748 </span> :
<span class="lineNum"> 749 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 750 </span><span class="lineCov"> 16469056 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 751 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 752 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 753 </span> :
<span class="lineNum"> 754 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));</span>
<span class="lineNum"> 755 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));</span>
<span class="lineNum"> 756 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 757 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 758 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);</span>
<span class="lineNum"> 759 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);</span>
<span class="lineNum"> 760 </span> :
<span class="lineNum"> 761 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));</span>
<span class="lineNum"> 762 </span><span class="lineCov"> 16469056 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));</span>
<span class="lineNum"> 763 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 764 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 765 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm6,%xmm3&quot;);</span>
<span class="lineNum"> 766 </span><span class="lineCov"> 16469056 : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);</span>
<span class="lineNum"> 767 </span> : }
<span class="lineNum"> 768 </span> :
<span class="lineNum"> 769 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 770 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 771 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[0][i]));</span>
<span class="lineNum"> 772 </span> :
<span class="lineNum"> 773 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 774 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);</span>
<span class="lineNum"> 775 </span><span class="lineCov"> 3302336 : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);</span>
<span class="lineNum"> 776 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 777 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);</span>
<span class="lineNum"> 778 </span> :
<span class="lineNum"> 779 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 780 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 781 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 782 </span><span class="lineCov"> 3302336 : asm volatile (&quot;pxor %xmm4,%xmm3&quot;);</span>
<span class="lineNum"> 783 </span> :
<span class="lineNum"> 784 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 785 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (q[i]));</span>
<span class="lineNum"> 786 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (r[i]));</span>
<span class="lineNum"> 787 </span><span class="lineCov"> 3302336 : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (s[i]));</span>
<span class="lineNum"> 788 </span> : }
<span class="lineNum"> 789 </span> :
<span class="lineNum"> 790 </span> : raid_sse_end();
<span class="lineNum"> 791 </span> : }
<span class="lineNum"> 792 </span> : #endif
<span class="lineNum"> 793 </span> :
<span class="lineNum"> 794 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 795 </span> : /*
<span class="lineNum"> 796 </span> : * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
<span class="lineNum"> 797 </span> : *
<span class="lineNum"> 798 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 799 </span> : */
<span class="lineNum"> 800 </span> : void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
<span class="lineNum"> 801 </span> : {
<span class="lineNum"> 802 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 803 </span> : uint8_t *p;
<span class="lineNum"> 804 </span> : uint8_t *q;
<span class="lineNum"> 805 </span> : uint8_t *r;
<span class="lineNum"> 806 </span> : uint8_t *s;
<span class="lineNum"> 807 </span> : int d, l;
<span class="lineNum"> 808 </span> : size_t i;
<span class="lineNum"> 809 </span> :
<span class="lineNum"> 810 </span> : l = nd - 1;
<span class="lineNum"> 811 </span> : p = v[nd];
<span class="lineNum"> 812 </span> : q = v[nd + 1];
<span class="lineNum"> 813 </span> : r = v[nd + 2];
<span class="lineNum"> 814 </span> : s = v[nd + 3];
<span class="lineNum"> 815 </span> :
<span class="lineNum"> 816 </span> : /* special case with only one data disk */
<span class="lineNum"> 817 </span> : if (l == 0) {
<span class="lineNum"> 818 </span> : for (i = 0; i &lt; 4; ++i)
<span class="lineNum"> 819 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 820 </span> : return;
<span class="lineNum"> 821 </span> : }
<span class="lineNum"> 822 </span> :
<span class="lineNum"> 823 </span> : raid_sse_begin();
<span class="lineNum"> 824 </span> :
<span class="lineNum"> 825 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 826 </span> : for (i = 0; i &lt; size; i += 32) {
<span class="lineNum"> 827 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 828 </span> : asm volatile (&quot;movdqa %0,%%xmm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 829 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 830 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (v[l][i + 16]));
<span class="lineNum"> 831 </span> :
<span class="lineNum"> 832 </span> : asm volatile (&quot;movdqa %xmm4,%xmm0&quot;);
<span class="lineNum"> 833 </span> : asm volatile (&quot;movdqa %xmm4,%xmm1&quot;);
<span class="lineNum"> 834 </span> : asm volatile (&quot;movdqa %xmm12,%xmm8&quot;);
<span class="lineNum"> 835 </span> : asm volatile (&quot;movdqa %xmm12,%xmm9&quot;);
<span class="lineNum"> 836 </span> :
<span class="lineNum"> 837 </span> : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);
<span class="lineNum"> 838 </span> : asm volatile (&quot;movdqa %xmm12,%xmm13&quot;);
<span class="lineNum"> 839 </span> : asm volatile (&quot;psrlw $4,%xmm5&quot;);
<span class="lineNum"> 840 </span> : asm volatile (&quot;psrlw $4,%xmm13&quot;);
<span class="lineNum"> 841 </span> : asm volatile (&quot;pand %xmm15,%xmm4&quot;);
<span class="lineNum"> 842 </span> : asm volatile (&quot;pand %xmm15,%xmm12&quot;);
<span class="lineNum"> 843 </span> : asm volatile (&quot;pand %xmm15,%xmm5&quot;);
<span class="lineNum"> 844 </span> : asm volatile (&quot;pand %xmm15,%xmm13&quot;);
<span class="lineNum"> 845 </span> :
<span class="lineNum"> 846 </span> : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 847 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 848 </span> : asm volatile (&quot;movdqa %xmm2,%xmm10&quot;);
<span class="lineNum"> 849 </span> : asm volatile (&quot;movdqa %xmm7,%xmm15&quot;);
<span class="lineNum"> 850 </span> : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);
<span class="lineNum"> 851 </span> : asm volatile (&quot;pshufb %xmm12,%xmm10&quot;);
<span class="lineNum"> 852 </span> : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);
<span class="lineNum"> 853 </span> : asm volatile (&quot;pshufb %xmm13,%xmm15&quot;);
<span class="lineNum"> 854 </span> : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);
<span class="lineNum"> 855 </span> : asm volatile (&quot;pxor %xmm15,%xmm10&quot;);
<span class="lineNum"> 856 </span> :
<span class="lineNum"> 857 </span> : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));
<span class="lineNum"> 858 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));
<span class="lineNum"> 859 </span> : asm volatile (&quot;movdqa %xmm3,%xmm11&quot;);
<span class="lineNum"> 860 </span> : asm volatile (&quot;movdqa %xmm7,%xmm15&quot;);
<span class="lineNum"> 861 </span> : asm volatile (&quot;pshufb %xmm4,%xmm3&quot;);
<span class="lineNum"> 862 </span> : asm volatile (&quot;pshufb %xmm12,%xmm11&quot;);
<span class="lineNum"> 863 </span> : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);
<span class="lineNum"> 864 </span> : asm volatile (&quot;pshufb %xmm13,%xmm15&quot;);
<span class="lineNum"> 865 </span> : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);
<span class="lineNum"> 866 </span> : asm volatile (&quot;pxor %xmm15,%xmm11&quot;);
<span class="lineNum"> 867 </span> :
<span class="lineNum"> 868 </span> : /* intermediate disks */
<span class="lineNum"> 869 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 870 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 871 </span> : asm volatile (&quot;movdqa %0,%%xmm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 872 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 873 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (v[d][i + 16]));
<span class="lineNum"> 874 </span> :
<span class="lineNum"> 875 </span> : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);
<span class="lineNum"> 876 </span> : asm volatile (&quot;pxor %xmm13,%xmm13&quot;);
<span class="lineNum"> 877 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);
<span class="lineNum"> 878 </span> : asm volatile (&quot;pcmpgtb %xmm9,%xmm13&quot;);
<span class="lineNum"> 879 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 880 </span> : asm volatile (&quot;paddb %xmm9,%xmm9&quot;);
<span class="lineNum"> 881 </span> : asm volatile (&quot;pand %xmm7,%xmm5&quot;);
<span class="lineNum"> 882 </span> : asm volatile (&quot;pand %xmm7,%xmm13&quot;);
<span class="lineNum"> 883 </span> : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);
<span class="lineNum"> 884 </span> : asm volatile (&quot;pxor %xmm13,%xmm9&quot;);
<span class="lineNum"> 885 </span> :
<span class="lineNum"> 886 </span> : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);
<span class="lineNum"> 887 </span> : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);
<span class="lineNum"> 888 </span> : asm volatile (&quot;pxor %xmm12,%xmm8&quot;);
<span class="lineNum"> 889 </span> : asm volatile (&quot;pxor %xmm12,%xmm9&quot;);
<span class="lineNum"> 890 </span> :
<span class="lineNum"> 891 </span> : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);
<span class="lineNum"> 892 </span> : asm volatile (&quot;movdqa %xmm12,%xmm13&quot;);
<span class="lineNum"> 893 </span> : asm volatile (&quot;psrlw $4,%xmm5&quot;);
<span class="lineNum"> 894 </span> : asm volatile (&quot;psrlw $4,%xmm13&quot;);
<span class="lineNum"> 895 </span> : asm volatile (&quot;pand %xmm15,%xmm4&quot;);
<span class="lineNum"> 896 </span> : asm volatile (&quot;pand %xmm15,%xmm12&quot;);
<span class="lineNum"> 897 </span> : asm volatile (&quot;pand %xmm15,%xmm5&quot;);
<span class="lineNum"> 898 </span> : asm volatile (&quot;pand %xmm15,%xmm13&quot;);
<span class="lineNum"> 899 </span> :
<span class="lineNum"> 900 </span> : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 901 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 902 </span> : asm volatile (&quot;movdqa %xmm6,%xmm14&quot;);
<span class="lineNum"> 903 </span> : asm volatile (&quot;movdqa %xmm7,%xmm15&quot;);
<span class="lineNum"> 904 </span> : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);
<span class="lineNum"> 905 </span> : asm volatile (&quot;pshufb %xmm12,%xmm14&quot;);
<span class="lineNum"> 906 </span> : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);
<span class="lineNum"> 907 </span> : asm volatile (&quot;pshufb %xmm13,%xmm15&quot;);
<span class="lineNum"> 908 </span> : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);
<span class="lineNum"> 909 </span> : asm volatile (&quot;pxor %xmm14,%xmm10&quot;);
<span class="lineNum"> 910 </span> : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);
<span class="lineNum"> 911 </span> : asm volatile (&quot;pxor %xmm15,%xmm10&quot;);
<span class="lineNum"> 912 </span> :
<span class="lineNum"> 913 </span> : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));
<span class="lineNum"> 914 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));
<span class="lineNum"> 915 </span> : asm volatile (&quot;movdqa %xmm6,%xmm14&quot;);
<span class="lineNum"> 916 </span> : asm volatile (&quot;movdqa %xmm7,%xmm15&quot;);
<span class="lineNum"> 917 </span> : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);
<span class="lineNum"> 918 </span> : asm volatile (&quot;pshufb %xmm12,%xmm14&quot;);
<span class="lineNum"> 919 </span> : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);
<span class="lineNum"> 920 </span> : asm volatile (&quot;pshufb %xmm13,%xmm15&quot;);
<span class="lineNum"> 921 </span> : asm volatile (&quot;pxor %xmm6,%xmm3&quot;);
<span class="lineNum"> 922 </span> : asm volatile (&quot;pxor %xmm14,%xmm11&quot;);
<span class="lineNum"> 923 </span> : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);
<span class="lineNum"> 924 </span> : asm volatile (&quot;pxor %xmm15,%xmm11&quot;);
<span class="lineNum"> 925 </span> : }
<span class="lineNum"> 926 </span> :
<span class="lineNum"> 927 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 928 </span> : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 929 </span> : asm volatile (&quot;movdqa %0,%%xmm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 930 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 931 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (v[0][i + 16]));
<span class="lineNum"> 932 </span> :
<span class="lineNum"> 933 </span> : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);
<span class="lineNum"> 934 </span> : asm volatile (&quot;pxor %xmm13,%xmm13&quot;);
<span class="lineNum"> 935 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm5&quot;);
<span class="lineNum"> 936 </span> : asm volatile (&quot;pcmpgtb %xmm9,%xmm13&quot;);
<span class="lineNum"> 937 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 938 </span> : asm volatile (&quot;paddb %xmm9,%xmm9&quot;);
<span class="lineNum"> 939 </span> : asm volatile (&quot;pand %xmm7,%xmm5&quot;);
<span class="lineNum"> 940 </span> : asm volatile (&quot;pand %xmm7,%xmm13&quot;);
<span class="lineNum"> 941 </span> : asm volatile (&quot;pxor %xmm5,%xmm1&quot;);
<span class="lineNum"> 942 </span> : asm volatile (&quot;pxor %xmm13,%xmm9&quot;);
<span class="lineNum"> 943 </span> :
<span class="lineNum"> 944 </span> : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);
<span class="lineNum"> 945 </span> : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);
<span class="lineNum"> 946 </span> : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);
<span class="lineNum"> 947 </span> : asm volatile (&quot;pxor %xmm4,%xmm3&quot;);
<span class="lineNum"> 948 </span> : asm volatile (&quot;pxor %xmm12,%xmm8&quot;);
<span class="lineNum"> 949 </span> : asm volatile (&quot;pxor %xmm12,%xmm9&quot;);
<span class="lineNum"> 950 </span> : asm volatile (&quot;pxor %xmm12,%xmm10&quot;);
<span class="lineNum"> 951 </span> : asm volatile (&quot;pxor %xmm12,%xmm11&quot;);
<span class="lineNum"> 952 </span> :
<span class="lineNum"> 953 </span> : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 954 </span> : asm volatile (&quot;movntdq %%xmm8,%0&quot; : &quot;=m&quot; (p[i + 16]));
<span class="lineNum"> 955 </span> : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 956 </span> : asm volatile (&quot;movntdq %%xmm9,%0&quot; : &quot;=m&quot; (q[i + 16]));
<span class="lineNum"> 957 </span> : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 958 </span> : asm volatile (&quot;movntdq %%xmm10,%0&quot; : &quot;=m&quot; (r[i + 16]));
<span class="lineNum"> 959 </span> : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (s[i]));
<span class="lineNum"> 960 </span> : asm volatile (&quot;movntdq %%xmm11,%0&quot; : &quot;=m&quot; (s[i + 16]));
<span class="lineNum"> 961 </span> : }
<span class="lineNum"> 962 </span> :
<span class="lineNum"> 963 </span> : raid_sse_end();
<span class="lineNum"> 964 </span> : }
<span class="lineNum"> 965 </span> : #endif
<span class="lineNum"> 966 </span> :
<span class="lineNum"> 967 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 968 </span> : /*
<span class="lineNum"> 969 </span> : * GEN4 (quad parity with Cauchy matrix) AVX2 implementation
<span class="lineNum"> 970 </span> : *
<span class="lineNum"> 971 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 972 </span> : */
<span class="lineNum"> 973 </span> : void raid_gen4_avx2ext(int nd, size_t size, void **vv)
<span class="lineNum"> 974 </span> : {
<span class="lineNum"> 975 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 976 </span> : uint8_t *p;
<span class="lineNum"> 977 </span> : uint8_t *q;
<span class="lineNum"> 978 </span> : uint8_t *r;
<span class="lineNum"> 979 </span> : uint8_t *s;
<span class="lineNum"> 980 </span> : int d, l;
<span class="lineNum"> 981 </span> : size_t i;
<span class="lineNum"> 982 </span> :
<span class="lineNum"> 983 </span> : l = nd - 1;
<span class="lineNum"> 984 </span> : p = v[nd];
<span class="lineNum"> 985 </span> : q = v[nd + 1];
<span class="lineNum"> 986 </span> : r = v[nd + 2];
<span class="lineNum"> 987 </span> : s = v[nd + 3];
<span class="lineNum"> 988 </span> :
<span class="lineNum"> 989 </span> : /* special case with only one data disk */
<span class="lineNum"> 990 </span> : if (l == 0) {
<span class="lineNum"> 991 </span> : for (i = 0; i &lt; 4; ++i)
<span class="lineNum"> 992 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 993 </span> : return;
<span class="lineNum"> 994 </span> : }
<span class="lineNum"> 995 </span> :
<span class="lineNum"> 996 </span> : raid_avx_begin();
<span class="lineNum"> 997 </span> :
<span class="lineNum"> 998 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 999 </span> : for (i = 0; i &lt; size; i += 64) {
<span class="lineNum"> 1000 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1001 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1002 </span> : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 1003 </span> : asm volatile (&quot;vmovdqa %0,%%ymm12&quot; : : &quot;m&quot; (v[l][i + 32]));
<span class="lineNum"> 1004 </span> :
<span class="lineNum"> 1005 </span> : asm volatile (&quot;vmovdqa %ymm4,%ymm0&quot;);
<span class="lineNum"> 1006 </span> : asm volatile (&quot;vmovdqa %ymm4,%ymm1&quot;);
<span class="lineNum"> 1007 </span> : asm volatile (&quot;vmovdqa %ymm12,%ymm8&quot;);
<span class="lineNum"> 1008 </span> : asm volatile (&quot;vmovdqa %ymm12,%ymm9&quot;);
<span class="lineNum"> 1009 </span> :
<span class="lineNum"> 1010 </span> : asm volatile (&quot;vpsrlw $4,%ymm4,%ymm5&quot;);
<span class="lineNum"> 1011 </span> : asm volatile (&quot;vpsrlw $4,%ymm12,%ymm13&quot;);
<span class="lineNum"> 1012 </span> : asm volatile (&quot;vpand %ymm15,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1013 </span> : asm volatile (&quot;vpand %ymm15,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1014 </span> : asm volatile (&quot;vpand %ymm15,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1015 </span> : asm volatile (&quot;vpand %ymm15,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1016 </span> :
<span class="lineNum"> 1017 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm10&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 1018 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 1019 </span> : asm volatile (&quot;vpshufb %ymm4,%ymm10,%ymm2&quot;);
<span class="lineNum"> 1020 </span> : asm volatile (&quot;vpshufb %ymm5,%ymm15,%ymm7&quot;);
<span class="lineNum"> 1021 </span> : asm volatile (&quot;vpshufb %ymm12,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1022 </span> : asm volatile (&quot;vpshufb %ymm13,%ymm15,%ymm15&quot;);
<span class="lineNum"> 1023 </span> : asm volatile (&quot;vpxor %ymm7,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1024 </span> : asm volatile (&quot;vpxor %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1025 </span> :
<span class="lineNum"> 1026 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm11&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));
<span class="lineNum"> 1027 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));
<span class="lineNum"> 1028 </span> : asm volatile (&quot;vpshufb %ymm4,%ymm11,%ymm3&quot;);
<span class="lineNum"> 1029 </span> : asm volatile (&quot;vpshufb %ymm5,%ymm15,%ymm7&quot;);
<span class="lineNum"> 1030 </span> : asm volatile (&quot;vpshufb %ymm12,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1031 </span> : asm volatile (&quot;vpshufb %ymm13,%ymm15,%ymm15&quot;);
<span class="lineNum"> 1032 </span> : asm volatile (&quot;vpxor %ymm7,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1033 </span> : asm volatile (&quot;vpxor %ymm15,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1034 </span> :
<span class="lineNum"> 1035 </span> : /* intermediate disks */
<span class="lineNum"> 1036 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 1037 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 1038 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1039 </span> : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 1040 </span> : asm volatile (&quot;vmovdqa %0,%%ymm12&quot; : : &quot;m&quot; (v[d][i + 32]));
<span class="lineNum"> 1041 </span> :
<span class="lineNum"> 1042 </span> : asm volatile (&quot;vpxor %ymm5,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1043 </span> : asm volatile (&quot;vpxor %ymm13,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1044 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1045 </span> : asm volatile (&quot;vpcmpgtb %ymm9,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1046 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1047 </span> : asm volatile (&quot;vpaddb %ymm9,%ymm9,%ymm9&quot;);
<span class="lineNum"> 1048 </span> : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1049 </span> : asm volatile (&quot;vpand %ymm7,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1050 </span> : asm volatile (&quot;vpxor %ymm5,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1051 </span> : asm volatile (&quot;vpxor %ymm13,%ymm9,%ymm9&quot;);
<span class="lineNum"> 1052 </span> :
<span class="lineNum"> 1053 </span> : asm volatile (&quot;vpxor %ymm4,%ymm0,%ymm0&quot;);
<span class="lineNum"> 1054 </span> : asm volatile (&quot;vpxor %ymm4,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1055 </span> : asm volatile (&quot;vpxor %ymm12,%ymm8,%ymm8&quot;);
<span class="lineNum"> 1056 </span> : asm volatile (&quot;vpxor %ymm12,%ymm9,%ymm9&quot;);
<span class="lineNum"> 1057 </span> :
<span class="lineNum"> 1058 </span> : asm volatile (&quot;vpsrlw $4,%ymm4,%ymm5&quot;);
<span class="lineNum"> 1059 </span> : asm volatile (&quot;vpsrlw $4,%ymm12,%ymm13&quot;);
<span class="lineNum"> 1060 </span> : asm volatile (&quot;vpand %ymm15,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1061 </span> : asm volatile (&quot;vpand %ymm15,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1062 </span> : asm volatile (&quot;vpand %ymm15,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1063 </span> : asm volatile (&quot;vpand %ymm15,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1064 </span> :
<span class="lineNum"> 1065 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm14&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 1066 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 1067 </span> : asm volatile (&quot;vpshufb %ymm4,%ymm14,%ymm6&quot;);
<span class="lineNum"> 1068 </span> : asm volatile (&quot;vpshufb %ymm5,%ymm15,%ymm7&quot;);
<span class="lineNum"> 1069 </span> : asm volatile (&quot;vpshufb %ymm12,%ymm14,%ymm14&quot;);
<span class="lineNum"> 1070 </span> : asm volatile (&quot;vpshufb %ymm13,%ymm15,%ymm15&quot;);
<span class="lineNum"> 1071 </span> : asm volatile (&quot;vpxor %ymm6,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1072 </span> : asm volatile (&quot;vpxor %ymm14,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1073 </span> : asm volatile (&quot;vpxor %ymm7,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1074 </span> : asm volatile (&quot;vpxor %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1075 </span> :
<span class="lineNum"> 1076 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm14&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));
<span class="lineNum"> 1077 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));
<span class="lineNum"> 1078 </span> : asm volatile (&quot;vpshufb %ymm4,%ymm14,%ymm6&quot;);
<span class="lineNum"> 1079 </span> : asm volatile (&quot;vpshufb %ymm5,%ymm15,%ymm7&quot;);
<span class="lineNum"> 1080 </span> : asm volatile (&quot;vpshufb %ymm12,%ymm14,%ymm14&quot;);
<span class="lineNum"> 1081 </span> : asm volatile (&quot;vpshufb %ymm13,%ymm15,%ymm15&quot;);
<span class="lineNum"> 1082 </span> : asm volatile (&quot;vpxor %ymm6,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1083 </span> : asm volatile (&quot;vpxor %ymm14,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1084 </span> : asm volatile (&quot;vpxor %ymm7,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1085 </span> : asm volatile (&quot;vpxor %ymm15,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1086 </span> : }
<span class="lineNum"> 1087 </span> :
<span class="lineNum"> 1088 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1089 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 1090 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1091 </span> : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 1092 </span> : asm volatile (&quot;vmovdqa %0,%%ymm12&quot; : : &quot;m&quot; (v[0][i + 32]));
<span class="lineNum"> 1093 </span> :
<span class="lineNum"> 1094 </span> : asm volatile (&quot;vpxor %ymm5,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1095 </span> : asm volatile (&quot;vpxor %ymm13,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1096 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1097 </span> : asm volatile (&quot;vpcmpgtb %ymm9,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1098 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1099 </span> : asm volatile (&quot;vpaddb %ymm9,%ymm9,%ymm9&quot;);
<span class="lineNum"> 1100 </span> : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1101 </span> : asm volatile (&quot;vpand %ymm7,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1102 </span> : asm volatile (&quot;vpxor %ymm5,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1103 </span> : asm volatile (&quot;vpxor %ymm13,%ymm9,%ymm9&quot;);
<span class="lineNum"> 1104 </span> :
<span class="lineNum"> 1105 </span> : asm volatile (&quot;vpxor %ymm4,%ymm0,%ymm0&quot;);
<span class="lineNum"> 1106 </span> : asm volatile (&quot;vpxor %ymm4,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1107 </span> : asm volatile (&quot;vpxor %ymm4,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1108 </span> : asm volatile (&quot;vpxor %ymm4,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1109 </span> : asm volatile (&quot;vpxor %ymm12,%ymm8,%ymm8&quot;);
<span class="lineNum"> 1110 </span> : asm volatile (&quot;vpxor %ymm12,%ymm9,%ymm9&quot;);
<span class="lineNum"> 1111 </span> : asm volatile (&quot;vpxor %ymm12,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1112 </span> : asm volatile (&quot;vpxor %ymm12,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1113 </span> :
<span class="lineNum"> 1114 </span> : asm volatile (&quot;vmovntdq %%ymm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 1115 </span> : asm volatile (&quot;vmovntdq %%ymm8,%0&quot; : &quot;=m&quot; (p[i + 32]));
<span class="lineNum"> 1116 </span> : asm volatile (&quot;vmovntdq %%ymm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 1117 </span> : asm volatile (&quot;vmovntdq %%ymm9,%0&quot; : &quot;=m&quot; (q[i + 32]));
<span class="lineNum"> 1118 </span> : asm volatile (&quot;vmovntdq %%ymm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 1119 </span> : asm volatile (&quot;vmovntdq %%ymm10,%0&quot; : &quot;=m&quot; (r[i + 32]));
<span class="lineNum"> 1120 </span> : asm volatile (&quot;vmovntdq %%ymm3,%0&quot; : &quot;=m&quot; (s[i]));
<span class="lineNum"> 1121 </span> : asm volatile (&quot;vmovntdq %%ymm11,%0&quot; : &quot;=m&quot; (s[i + 32]));
<span class="lineNum"> 1122 </span> : }
<span class="lineNum"> 1123 </span> :
<span class="lineNum"> 1124 </span> : raid_avx_end();
<span class="lineNum"> 1125 </span> : }
<span class="lineNum"> 1126 </span> : #endif
<span class="lineNum"> 1127 </span> :
<span class="lineNum"> 1128 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 1129 </span> : /*
<a name="1130"><span class="lineNum"> 1130 </span> : * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation</a>
<span class="lineNum"> 1131 </span> : */
<span class="lineNum"> 1132 </span><span class="lineCov"> 50654 : void raid_gen5_ssse3(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 1133 </span> : {
<span class="lineNum"> 1134 </span><span class="lineCov"> 50654 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 1135 </span> : uint8_t *p;
<span class="lineNum"> 1136 </span> : uint8_t *q;
<span class="lineNum"> 1137 </span> : uint8_t *r;
<span class="lineNum"> 1138 </span> : uint8_t *s;
<span class="lineNum"> 1139 </span> : uint8_t *t;
<span class="lineNum"> 1140 </span> : int d, l;
<span class="lineNum"> 1141 </span> : size_t i;
<span class="lineNum"> 1142 </span> : uint8_t buffer[16+16];
<span class="lineNum"> 1143 </span><span class="lineCov"> 50654 : uint8_t *pd = __align_ptr(buffer, 16);</span>
<span class="lineNum"> 1144 </span> :
<span class="lineNum"> 1145 </span><span class="lineCov"> 50654 : l = nd - 1;</span>
<span class="lineNum"> 1146 </span><span class="lineCov"> 50654 : p = v[nd];</span>
<span class="lineNum"> 1147 </span><span class="lineCov"> 50654 : q = v[nd + 1];</span>
<span class="lineNum"> 1148 </span><span class="lineCov"> 50654 : r = v[nd + 2];</span>
<span class="lineNum"> 1149 </span><span class="lineCov"> 50654 : s = v[nd + 3];</span>
<span class="lineNum"> 1150 </span><span class="lineCov"> 50654 : t = v[nd + 4];</span>
<span class="lineNum"> 1151 </span> :
<span class="lineNum"> 1152 </span> : /* special case with only one data disk */
<span class="lineNum"> 1153 </span><span class="lineCov"> 50654 : if (l == 0) {</span>
<span class="lineNum"> 1154 </span><span class="lineCov"> 6 : for (i = 0; i &lt; 5; ++i)</span>
<span class="lineNum"> 1155 </span><span class="lineCov"> 5 : memcpy(v[1 + i], v[0], size);</span>
<span class="lineNum"> 1156 </span><span class="lineCov"> 50655 : return;</span>
<span class="lineNum"> 1157 </span> : }
<span class="lineNum"> 1158 </span> :
<span class="lineNum"> 1159 </span> : raid_sse_begin();
<span class="lineNum"> 1160 </span> :
<span class="lineNum"> 1161 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 1162 </span><span class="lineCov"> 3937517 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 1163 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1164 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 1165 </span> :
<span class="lineNum"> 1166 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 1167 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %%xmm4,%0&quot; : &quot;=m&quot; (pd[0]));</span>
<span class="lineNum"> 1168 </span> :
<span class="lineNum"> 1169 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 1170 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 1171 </span><span class="lineCov"> 3886864 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 1172 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 1173 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 1174 </span> :
<span class="lineNum"> 1175 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));</span>
<span class="lineNum"> 1176 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));</span>
<span class="lineNum"> 1177 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pshufb %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 1178 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1179 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm7,%xmm1&quot;);</span>
<span class="lineNum"> 1180 </span> :
<span class="lineNum"> 1181 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));</span>
<span class="lineNum"> 1182 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));</span>
<span class="lineNum"> 1183 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 1184 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1185 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);</span>
<span class="lineNum"> 1186 </span> :
<span class="lineNum"> 1187 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfgenpshufb[l][2][0][0]));</span>
<span class="lineNum"> 1188 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][2][1][0]));</span>
<span class="lineNum"> 1189 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pshufb %xmm4,%xmm3&quot;);</span>
<span class="lineNum"> 1190 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1191 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);</span>
<span class="lineNum"> 1192 </span> :
<span class="lineNum"> 1193 </span> : /* intermediate disks */
<span class="lineNum"> 1194 </span><span class="lineCov"> 23263984 : for (d = l - 1; d &gt; 0; --d) {</span>
<span class="lineNum"> 1195 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 1196 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (pd[0]));</span>
<span class="lineNum"> 1197 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 1198 </span> :
<span class="lineNum"> 1199 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 1200 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pcmpgtb %xmm0,%xmm5&quot;);</span>
<span class="lineNum"> 1201 </span><span class="lineCov"> 19377120 : asm volatile (&quot;paddb %xmm0,%xmm0&quot;);</span>
<span class="lineNum"> 1202 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 1203 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm5,%xmm0&quot;);</span>
<span class="lineNum"> 1204 </span> :
<span class="lineNum"> 1205 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 1206 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1207 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %%xmm6,%0&quot; : &quot;=m&quot; (pd[0]));</span>
<span class="lineNum"> 1208 </span> :
<span class="lineNum"> 1209 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 1210 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 1211 </span><span class="lineCov"> 19377120 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 1212 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 1213 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 1214 </span> :
<span class="lineNum"> 1215 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));</span>
<span class="lineNum"> 1216 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));</span>
<span class="lineNum"> 1217 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1218 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1219 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm6,%xmm1&quot;);</span>
<span class="lineNum"> 1220 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm7,%xmm1&quot;);</span>
<span class="lineNum"> 1221 </span> :
<span class="lineNum"> 1222 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));</span>
<span class="lineNum"> 1223 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));</span>
<span class="lineNum"> 1224 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1225 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1226 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);</span>
<span class="lineNum"> 1227 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);</span>
<span class="lineNum"> 1228 </span> :
<span class="lineNum"> 1229 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][2][0][0]));</span>
<span class="lineNum"> 1230 </span><span class="lineCov"> 19377120 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][2][1][0]));</span>
<span class="lineNum"> 1231 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1232 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1233 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm6,%xmm3&quot;);</span>
<span class="lineNum"> 1234 </span><span class="lineCov"> 19377120 : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);</span>
<span class="lineNum"> 1235 </span> : }
<span class="lineNum"> 1236 </span> :
<span class="lineNum"> 1237 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1238 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[0][i]));</span>
<span class="lineNum"> 1239 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (pd[0]));</span>
<span class="lineNum"> 1240 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 1241 </span> :
<span class="lineNum"> 1242 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm5,%xmm5&quot;);</span>
<span class="lineNum"> 1243 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pcmpgtb %xmm0,%xmm5&quot;);</span>
<span class="lineNum"> 1244 </span><span class="lineCov"> 3886864 : asm volatile (&quot;paddb %xmm0,%xmm0&quot;);</span>
<span class="lineNum"> 1245 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 1246 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm5,%xmm0&quot;);</span>
<span class="lineNum"> 1247 </span> :
<span class="lineNum"> 1248 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 1249 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 1250 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 1251 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm4,%xmm3&quot;);</span>
<span class="lineNum"> 1252 </span><span class="lineCov"> 3886864 : asm volatile (&quot;pxor %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1253 </span> :
<span class="lineNum"> 1254 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movntdq %%xmm6,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 1255 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (q[i]));</span>
<span class="lineNum"> 1256 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (r[i]));</span>
<span class="lineNum"> 1257 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (s[i]));</span>
<span class="lineNum"> 1258 </span><span class="lineCov"> 3886864 : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (t[i]));</span>
<span class="lineNum"> 1259 </span> : }
<span class="lineNum"> 1260 </span> :
<span class="lineNum"> 1261 </span> : raid_sse_end();
<span class="lineNum"> 1262 </span> : }
<span class="lineNum"> 1263 </span> : #endif
<span class="lineNum"> 1264 </span> :
<span class="lineNum"> 1265 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 1266 </span> : /*
<span class="lineNum"> 1267 </span> : * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
<span class="lineNum"> 1268 </span> : *
<span class="lineNum"> 1269 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 1270 </span> : */
<span class="lineNum"> 1271 </span> : void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
<span class="lineNum"> 1272 </span> : {
<span class="lineNum"> 1273 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 1274 </span> : uint8_t *p;
<span class="lineNum"> 1275 </span> : uint8_t *q;
<span class="lineNum"> 1276 </span> : uint8_t *r;
<span class="lineNum"> 1277 </span> : uint8_t *s;
<span class="lineNum"> 1278 </span> : uint8_t *t;
<span class="lineNum"> 1279 </span> : int d, l;
<span class="lineNum"> 1280 </span> : size_t i;
<span class="lineNum"> 1281 </span> :
<span class="lineNum"> 1282 </span> : l = nd - 1;
<span class="lineNum"> 1283 </span> : p = v[nd];
<span class="lineNum"> 1284 </span> : q = v[nd + 1];
<span class="lineNum"> 1285 </span> : r = v[nd + 2];
<span class="lineNum"> 1286 </span> : s = v[nd + 3];
<span class="lineNum"> 1287 </span> : t = v[nd + 4];
<span class="lineNum"> 1288 </span> :
<span class="lineNum"> 1289 </span> : /* special case with only one data disk */
<span class="lineNum"> 1290 </span> : if (l == 0) {
<span class="lineNum"> 1291 </span> : for (i = 0; i &lt; 5; ++i)
<span class="lineNum"> 1292 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 1293 </span> : return;
<span class="lineNum"> 1294 </span> : }
<span class="lineNum"> 1295 </span> :
<span class="lineNum"> 1296 </span> : raid_sse_begin();
<span class="lineNum"> 1297 </span> :
<span class="lineNum"> 1298 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 1299 </span> : asm volatile (&quot;movdqa %0,%%xmm14&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 1300 </span> : asm volatile (&quot;movdqa %0,%%xmm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1301 </span> :
<span class="lineNum"> 1302 </span> : for (i = 0; i &lt; size; i += 16) {
<span class="lineNum"> 1303 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1304 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 1305 </span> :
<span class="lineNum"> 1306 </span> : asm volatile (&quot;movdqa %xmm10,%xmm0&quot;);
<span class="lineNum"> 1307 </span> : asm volatile (&quot;movdqa %xmm10,%xmm1&quot;);
<span class="lineNum"> 1308 </span> :
<span class="lineNum"> 1309 </span> : asm volatile (&quot;movdqa %xmm10,%xmm11&quot;);
<span class="lineNum"> 1310 </span> : asm volatile (&quot;psrlw $4,%xmm11&quot;);
<span class="lineNum"> 1311 </span> : asm volatile (&quot;pand %xmm15,%xmm10&quot;);
<span class="lineNum"> 1312 </span> : asm volatile (&quot;pand %xmm15,%xmm11&quot;);
<span class="lineNum"> 1313 </span> :
<span class="lineNum"> 1314 </span> : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 1315 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 1316 </span> : asm volatile (&quot;pshufb %xmm10,%xmm2&quot;);
<span class="lineNum"> 1317 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1318 </span> : asm volatile (&quot;pxor %xmm13,%xmm2&quot;);
<span class="lineNum"> 1319 </span> :
<span class="lineNum"> 1320 </span> : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));
<span class="lineNum"> 1321 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));
<span class="lineNum"> 1322 </span> : asm volatile (&quot;pshufb %xmm10,%xmm3&quot;);
<span class="lineNum"> 1323 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1324 </span> : asm volatile (&quot;pxor %xmm13,%xmm3&quot;);
<span class="lineNum"> 1325 </span> :
<span class="lineNum"> 1326 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (gfgenpshufb[l][2][0][0]));
<span class="lineNum"> 1327 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][2][1][0]));
<span class="lineNum"> 1328 </span> : asm volatile (&quot;pshufb %xmm10,%xmm4&quot;);
<span class="lineNum"> 1329 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1330 </span> : asm volatile (&quot;pxor %xmm13,%xmm4&quot;);
<span class="lineNum"> 1331 </span> :
<span class="lineNum"> 1332 </span> : /* intermediate disks */
<span class="lineNum"> 1333 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 1334 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 1335 </span> :
<span class="lineNum"> 1336 </span> : asm volatile (&quot;pxor %xmm11,%xmm11&quot;);
<span class="lineNum"> 1337 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm11&quot;);
<span class="lineNum"> 1338 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 1339 </span> : asm volatile (&quot;pand %xmm14,%xmm11&quot;);
<span class="lineNum"> 1340 </span> : asm volatile (&quot;pxor %xmm11,%xmm1&quot;);
<span class="lineNum"> 1341 </span> :
<span class="lineNum"> 1342 </span> : asm volatile (&quot;pxor %xmm10,%xmm0&quot;);
<span class="lineNum"> 1343 </span> : asm volatile (&quot;pxor %xmm10,%xmm1&quot;);
<span class="lineNum"> 1344 </span> :
<span class="lineNum"> 1345 </span> : asm volatile (&quot;movdqa %xmm10,%xmm11&quot;);
<span class="lineNum"> 1346 </span> : asm volatile (&quot;psrlw $4,%xmm11&quot;);
<span class="lineNum"> 1347 </span> : asm volatile (&quot;pand %xmm15,%xmm10&quot;);
<span class="lineNum"> 1348 </span> : asm volatile (&quot;pand %xmm15,%xmm11&quot;);
<span class="lineNum"> 1349 </span> :
<span class="lineNum"> 1350 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 1351 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 1352 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1353 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1354 </span> : asm volatile (&quot;pxor %xmm12,%xmm2&quot;);
<span class="lineNum"> 1355 </span> : asm volatile (&quot;pxor %xmm13,%xmm2&quot;);
<span class="lineNum"> 1356 </span> :
<span class="lineNum"> 1357 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));
<span class="lineNum"> 1358 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));
<span class="lineNum"> 1359 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1360 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1361 </span> : asm volatile (&quot;pxor %xmm12,%xmm3&quot;);
<span class="lineNum"> 1362 </span> : asm volatile (&quot;pxor %xmm13,%xmm3&quot;);
<span class="lineNum"> 1363 </span> :
<span class="lineNum"> 1364 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][2][0][0]));
<span class="lineNum"> 1365 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][2][1][0]));
<span class="lineNum"> 1366 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1367 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1368 </span> : asm volatile (&quot;pxor %xmm12,%xmm4&quot;);
<span class="lineNum"> 1369 </span> : asm volatile (&quot;pxor %xmm13,%xmm4&quot;);
<span class="lineNum"> 1370 </span> : }
<span class="lineNum"> 1371 </span> :
<span class="lineNum"> 1372 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1373 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 1374 </span> :
<span class="lineNum"> 1375 </span> : asm volatile (&quot;pxor %xmm11,%xmm11&quot;);
<span class="lineNum"> 1376 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm11&quot;);
<span class="lineNum"> 1377 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 1378 </span> : asm volatile (&quot;pand %xmm14,%xmm11&quot;);
<span class="lineNum"> 1379 </span> : asm volatile (&quot;pxor %xmm11,%xmm1&quot;);
<span class="lineNum"> 1380 </span> :
<span class="lineNum"> 1381 </span> : asm volatile (&quot;pxor %xmm10,%xmm0&quot;);
<span class="lineNum"> 1382 </span> : asm volatile (&quot;pxor %xmm10,%xmm1&quot;);
<span class="lineNum"> 1383 </span> : asm volatile (&quot;pxor %xmm10,%xmm2&quot;);
<span class="lineNum"> 1384 </span> : asm volatile (&quot;pxor %xmm10,%xmm3&quot;);
<span class="lineNum"> 1385 </span> : asm volatile (&quot;pxor %xmm10,%xmm4&quot;);
<span class="lineNum"> 1386 </span> :
<span class="lineNum"> 1387 </span> : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 1388 </span> : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 1389 </span> : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 1390 </span> : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (s[i]));
<span class="lineNum"> 1391 </span> : asm volatile (&quot;movntdq %%xmm4,%0&quot; : &quot;=m&quot; (t[i]));
<span class="lineNum"> 1392 </span> : }
<span class="lineNum"> 1393 </span> :
<span class="lineNum"> 1394 </span> : raid_sse_end();
<span class="lineNum"> 1395 </span> : }
<span class="lineNum"> 1396 </span> : #endif
<span class="lineNum"> 1397 </span> :
<span class="lineNum"> 1398 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 1399 </span> : /*
<span class="lineNum"> 1400 </span> : * GEN5 (penta parity with Cauchy matrix) AVX2 implementation
<span class="lineNum"> 1401 </span> : *
<span class="lineNum"> 1402 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 1403 </span> : */
<span class="lineNum"> 1404 </span> : void raid_gen5_avx2ext(int nd, size_t size, void **vv)
<span class="lineNum"> 1405 </span> : {
<span class="lineNum"> 1406 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 1407 </span> : uint8_t *p;
<span class="lineNum"> 1408 </span> : uint8_t *q;
<span class="lineNum"> 1409 </span> : uint8_t *r;
<span class="lineNum"> 1410 </span> : uint8_t *s;
<span class="lineNum"> 1411 </span> : uint8_t *t;
<span class="lineNum"> 1412 </span> : int d, l;
<span class="lineNum"> 1413 </span> : size_t i;
<span class="lineNum"> 1414 </span> :
<span class="lineNum"> 1415 </span> : l = nd - 1;
<span class="lineNum"> 1416 </span> : p = v[nd];
<span class="lineNum"> 1417 </span> : q = v[nd + 1];
<span class="lineNum"> 1418 </span> : r = v[nd + 2];
<span class="lineNum"> 1419 </span> : s = v[nd + 3];
<span class="lineNum"> 1420 </span> : t = v[nd + 4];
<span class="lineNum"> 1421 </span> :
<span class="lineNum"> 1422 </span> : /* special case with only one data disk */
<span class="lineNum"> 1423 </span> : if (l == 0) {
<span class="lineNum"> 1424 </span> : for (i = 0; i &lt; 5; ++i)
<span class="lineNum"> 1425 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 1426 </span> : return;
<span class="lineNum"> 1427 </span> : }
<span class="lineNum"> 1428 </span> :
<span class="lineNum"> 1429 </span> : raid_avx_begin();
<span class="lineNum"> 1430 </span> :
<span class="lineNum"> 1431 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 1432 </span> : asm volatile (&quot;vpxor %ymm8,%ymm8,%ymm8&quot;);
<span class="lineNum"> 1433 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm14&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 1434 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1435 </span> :
<span class="lineNum"> 1436 </span> : for (i = 0; i &lt; size; i += 32) {
<span class="lineNum"> 1437 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1438 </span> : asm volatile (&quot;vmovdqa %0,%%ymm10&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 1439 </span> :
<span class="lineNum"> 1440 </span> : asm volatile (&quot;vmovdqa %ymm10,%ymm0&quot;);
<span class="lineNum"> 1441 </span> : asm volatile (&quot;vmovdqa %ymm10,%ymm1&quot;);
<span class="lineNum"> 1442 </span> :
<span class="lineNum"> 1443 </span> : asm volatile (&quot;vpsrlw $4,%ymm10,%ymm11&quot;);
<span class="lineNum"> 1444 </span> : asm volatile (&quot;vpand %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1445 </span> : asm volatile (&quot;vpand %ymm15,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1446 </span> :
<span class="lineNum"> 1447 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 1448 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 1449 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1450 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1451 </span> : asm volatile (&quot;vpxor %ymm13,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1452 </span> :
<span class="lineNum"> 1453 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));
<span class="lineNum"> 1454 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));
<span class="lineNum"> 1455 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1456 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1457 </span> : asm volatile (&quot;vpxor %ymm13,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1458 </span> :
<span class="lineNum"> 1459 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm4&quot; : : &quot;m&quot; (gfgenpshufb[l][2][0][0]));
<span class="lineNum"> 1460 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][2][1][0]));
<span class="lineNum"> 1461 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1462 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1463 </span> : asm volatile (&quot;vpxor %ymm13,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1464 </span> :
<span class="lineNum"> 1465 </span> : /* intermediate disks */
<span class="lineNum"> 1466 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 1467 </span> : asm volatile (&quot;vmovdqa %0,%%ymm10&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 1468 </span> :
<span class="lineNum"> 1469 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm8,%ymm11&quot;);
<span class="lineNum"> 1470 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1471 </span> : asm volatile (&quot;vpand %ymm14,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1472 </span> : asm volatile (&quot;vpxor %ymm11,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1473 </span> :
<span class="lineNum"> 1474 </span> : asm volatile (&quot;vpxor %ymm10,%ymm0,%ymm0&quot;);
<span class="lineNum"> 1475 </span> : asm volatile (&quot;vpxor %ymm10,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1476 </span> :
<span class="lineNum"> 1477 </span> : asm volatile (&quot;vpsrlw $4,%ymm10,%ymm11&quot;);
<span class="lineNum"> 1478 </span> : asm volatile (&quot;vpand %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1479 </span> : asm volatile (&quot;vpand %ymm15,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1480 </span> :
<span class="lineNum"> 1481 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 1482 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 1483 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1484 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1485 </span> : asm volatile (&quot;vpxor %ymm12,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1486 </span> : asm volatile (&quot;vpxor %ymm13,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1487 </span> :
<span class="lineNum"> 1488 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));
<span class="lineNum"> 1489 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));
<span class="lineNum"> 1490 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1491 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1492 </span> : asm volatile (&quot;vpxor %ymm12,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1493 </span> : asm volatile (&quot;vpxor %ymm13,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1494 </span> :
<span class="lineNum"> 1495 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][2][0][0]));
<span class="lineNum"> 1496 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][2][1][0]));
<span class="lineNum"> 1497 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1498 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1499 </span> : asm volatile (&quot;vpxor %ymm12,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1500 </span> : asm volatile (&quot;vpxor %ymm13,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1501 </span> : }
<span class="lineNum"> 1502 </span> :
<span class="lineNum"> 1503 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1504 </span> : asm volatile (&quot;vmovdqa %0,%%ymm10&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 1505 </span> :
<span class="lineNum"> 1506 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm8,%ymm11&quot;);
<span class="lineNum"> 1507 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1508 </span> : asm volatile (&quot;vpand %ymm14,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1509 </span> : asm volatile (&quot;vpxor %ymm11,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1510 </span> :
<span class="lineNum"> 1511 </span> : asm volatile (&quot;vpxor %ymm10,%ymm0,%ymm0&quot;);
<span class="lineNum"> 1512 </span> : asm volatile (&quot;vpxor %ymm10,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1513 </span> : asm volatile (&quot;vpxor %ymm10,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1514 </span> : asm volatile (&quot;vpxor %ymm10,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1515 </span> : asm volatile (&quot;vpxor %ymm10,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1516 </span> :
<span class="lineNum"> 1517 </span> : asm volatile (&quot;vmovntdq %%ymm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 1518 </span> : asm volatile (&quot;vmovntdq %%ymm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 1519 </span> : asm volatile (&quot;vmovntdq %%ymm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 1520 </span> : asm volatile (&quot;vmovntdq %%ymm3,%0&quot; : &quot;=m&quot; (s[i]));
<span class="lineNum"> 1521 </span> : asm volatile (&quot;vmovntdq %%ymm4,%0&quot; : &quot;=m&quot; (t[i]));
<span class="lineNum"> 1522 </span> : }
<span class="lineNum"> 1523 </span> :
<span class="lineNum"> 1524 </span> : raid_avx_end();
<span class="lineNum"> 1525 </span> : }
<span class="lineNum"> 1526 </span> : #endif
<span class="lineNum"> 1527 </span> :
<span class="lineNum"> 1528 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 1529 </span> : /*
<a name="1530"><span class="lineNum"> 1530 </span> : * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation</a>
<span class="lineNum"> 1531 </span> : */
<span class="lineNum"> 1532 </span><span class="lineCov"> 564177 : void raid_gen6_ssse3(int nd, size_t size, void **vv)</span>
<span class="lineNum"> 1533 </span> : {
<span class="lineNum"> 1534 </span><span class="lineCov"> 564177 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 1535 </span> : uint8_t *p;
<span class="lineNum"> 1536 </span> : uint8_t *q;
<span class="lineNum"> 1537 </span> : uint8_t *r;
<span class="lineNum"> 1538 </span> : uint8_t *s;
<span class="lineNum"> 1539 </span> : uint8_t *t;
<span class="lineNum"> 1540 </span> : uint8_t *u;
<span class="lineNum"> 1541 </span> : int d, l;
<span class="lineNum"> 1542 </span> : size_t i;
<span class="lineNum"> 1543 </span> : uint8_t buffer[2*16+16];
<span class="lineNum"> 1544 </span><span class="lineCov"> 564177 : uint8_t *pd = __align_ptr(buffer, 16);</span>
<span class="lineNum"> 1545 </span> :
<span class="lineNum"> 1546 </span><span class="lineCov"> 564177 : l = nd - 1;</span>
<span class="lineNum"> 1547 </span><span class="lineCov"> 564177 : p = v[nd];</span>
<span class="lineNum"> 1548 </span><span class="lineCov"> 564177 : q = v[nd + 1];</span>
<span class="lineNum"> 1549 </span><span class="lineCov"> 564177 : r = v[nd + 2];</span>
<span class="lineNum"> 1550 </span><span class="lineCov"> 564177 : s = v[nd + 3];</span>
<span class="lineNum"> 1551 </span><span class="lineCov"> 564177 : t = v[nd + 4];</span>
<span class="lineNum"> 1552 </span><span class="lineCov"> 564177 : u = v[nd + 5];</span>
<span class="lineNum"> 1553 </span> :
<span class="lineNum"> 1554 </span> : /* special case with only one data disk */
<span class="lineNum"> 1555 </span><span class="lineCov"> 564177 : if (l == 0) {</span>
<span class="lineNum"> 1556 </span><span class="lineCov"> 7 : for (i = 0; i &lt; 6; ++i)</span>
<span class="lineNum"> 1557 </span><span class="lineCov"> 6 : memcpy(v[1 + i], v[0], size);</span>
<span class="lineNum"> 1558 </span><span class="lineCov"> 564178 : return;</span>
<span class="lineNum"> 1559 </span> : }
<span class="lineNum"> 1560 </span> :
<span class="lineNum"> 1561 </span> : raid_sse_begin();
<span class="lineNum"> 1562 </span> :
<span class="lineNum"> 1563 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 1564 </span><span class="lineCov"> 36098080 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 1565 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1566 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[l][i]));</span>
<span class="lineNum"> 1567 </span> :
<span class="lineNum"> 1568 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %%xmm4,%0&quot; : &quot;=m&quot; (pd[0]));</span>
<span class="lineNum"> 1569 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %%xmm4,%0&quot; : &quot;=m&quot; (pd[16]));</span>
<span class="lineNum"> 1570 </span> :
<span class="lineNum"> 1571 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 1572 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 1573 </span><span class="lineCov"> 35533904 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 1574 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 1575 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 1576 </span> :
<span class="lineNum"> 1577 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));</span>
<span class="lineNum"> 1578 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));</span>
<span class="lineNum"> 1579 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 1580 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1581 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm7,%xmm0&quot;);</span>
<span class="lineNum"> 1582 </span> :
<span class="lineNum"> 1583 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));</span>
<span class="lineNum"> 1584 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));</span>
<span class="lineNum"> 1585 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 1586 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1587 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm7,%xmm1&quot;);</span>
<span class="lineNum"> 1588 </span> :
<span class="lineNum"> 1589 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][2][0][0]));</span>
<span class="lineNum"> 1590 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][2][1][0]));</span>
<span class="lineNum"> 1591 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 1592 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1593 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);</span>
<span class="lineNum"> 1594 </span> :
<span class="lineNum"> 1595 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfgenpshufb[l][3][0][0]));</span>
<span class="lineNum"> 1596 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[l][3][1][0]));</span>
<span class="lineNum"> 1597 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm4,%xmm3&quot;);</span>
<span class="lineNum"> 1598 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1599 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);</span>
<span class="lineNum"> 1600 </span> :
<span class="lineNum"> 1601 </span> : /* intermediate disks */
<span class="lineNum"> 1602 </span><span class="lineCov"> 183674416 : for (d = l - 1; d &gt; 0; --d) {</span>
<span class="lineNum"> 1603 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm5&quot; : : &quot;m&quot; (pd[0]));</span>
<span class="lineNum"> 1604 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (pd[16]));</span>
<span class="lineNum"> 1605 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 1606 </span> :
<span class="lineNum"> 1607 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm4,%xmm4&quot;);</span>
<span class="lineNum"> 1608 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pcmpgtb %xmm6,%xmm4&quot;);</span>
<span class="lineNum"> 1609 </span><span class="lineCov"> 148140512 : asm volatile (&quot;paddb %xmm6,%xmm6&quot;);</span>
<span class="lineNum"> 1610 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 1611 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1612 </span> :
<span class="lineNum"> 1613 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[d][i]));</span>
<span class="lineNum"> 1614 </span> :
<span class="lineNum"> 1615 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 1616 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1617 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %%xmm5,%0&quot; : &quot;=m&quot; (pd[0]));</span>
<span class="lineNum"> 1618 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %%xmm6,%0&quot; : &quot;=m&quot; (pd[16]));</span>
<span class="lineNum"> 1619 </span> :
<span class="lineNum"> 1620 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 1621 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 1622 </span><span class="lineCov"> 148140512 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 1623 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 1624 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 1625 </span> :
<span class="lineNum"> 1626 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));</span>
<span class="lineNum"> 1627 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));</span>
<span class="lineNum"> 1628 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1629 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1630 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm6,%xmm0&quot;);</span>
<span class="lineNum"> 1631 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm7,%xmm0&quot;);</span>
<span class="lineNum"> 1632 </span> :
<span class="lineNum"> 1633 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));</span>
<span class="lineNum"> 1634 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));</span>
<span class="lineNum"> 1635 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1636 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1637 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm6,%xmm1&quot;);</span>
<span class="lineNum"> 1638 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm7,%xmm1&quot;);</span>
<span class="lineNum"> 1639 </span> :
<span class="lineNum"> 1640 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][2][0][0]));</span>
<span class="lineNum"> 1641 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][2][1][0]));</span>
<span class="lineNum"> 1642 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1643 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1644 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm6,%xmm2&quot;);</span>
<span class="lineNum"> 1645 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm7,%xmm2&quot;);</span>
<span class="lineNum"> 1646 </span> :
<span class="lineNum"> 1647 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (gfgenpshufb[d][3][0][0]));</span>
<span class="lineNum"> 1648 </span><span class="lineCov"> 148140512 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfgenpshufb[d][3][1][0]));</span>
<span class="lineNum"> 1649 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1650 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pshufb %xmm5,%xmm7&quot;);</span>
<span class="lineNum"> 1651 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm6,%xmm3&quot;);</span>
<span class="lineNum"> 1652 </span><span class="lineCov"> 148140512 : asm volatile (&quot;pxor %xmm7,%xmm3&quot;);</span>
<span class="lineNum"> 1653 </span> : }
<span class="lineNum"> 1654 </span> :
<span class="lineNum"> 1655 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1656 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm5&quot; : : &quot;m&quot; (pd[0]));</span>
<span class="lineNum"> 1657 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm6&quot; : : &quot;m&quot; (pd[16]));</span>
<span class="lineNum"> 1658 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.poly[0]));</span>
<span class="lineNum"> 1659 </span> :
<span class="lineNum"> 1660 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm4&quot;);</span>
<span class="lineNum"> 1661 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pcmpgtb %xmm6,%xmm4&quot;);</span>
<span class="lineNum"> 1662 </span><span class="lineCov"> 35533904 : asm volatile (&quot;paddb %xmm6,%xmm6&quot;);</span>
<span class="lineNum"> 1663 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 1664 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1665 </span> :
<span class="lineNum"> 1666 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (v[0][i]));</span>
<span class="lineNum"> 1667 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm0&quot;);</span>
<span class="lineNum"> 1668 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm1&quot;);</span>
<span class="lineNum"> 1669 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 1670 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm3&quot;);</span>
<span class="lineNum"> 1671 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 1672 </span><span class="lineCov"> 35533904 : asm volatile (&quot;pxor %xmm4,%xmm6&quot;);</span>
<span class="lineNum"> 1673 </span> :
<span class="lineNum"> 1674 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movntdq %%xmm5,%0&quot; : &quot;=m&quot; (p[i]));</span>
<span class="lineNum"> 1675 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movntdq %%xmm6,%0&quot; : &quot;=m&quot; (q[i]));</span>
<span class="lineNum"> 1676 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (r[i]));</span>
<span class="lineNum"> 1677 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (s[i]));</span>
<span class="lineNum"> 1678 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (t[i]));</span>
<span class="lineNum"> 1679 </span><span class="lineCov"> 35533904 : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (u[i]));</span>
<span class="lineNum"> 1680 </span> : }
<span class="lineNum"> 1681 </span> :
<span class="lineNum"> 1682 </span> : raid_sse_end();
<span class="lineNum"> 1683 </span> : }
<span class="lineNum"> 1684 </span> : #endif
<span class="lineNum"> 1685 </span> :
<span class="lineNum"> 1686 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 1687 </span> : /*
<span class="lineNum"> 1688 </span> : * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
<span class="lineNum"> 1689 </span> : *
<span class="lineNum"> 1690 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 1691 </span> : */
<span class="lineNum"> 1692 </span> : void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
<span class="lineNum"> 1693 </span> : {
<span class="lineNum"> 1694 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 1695 </span> : uint8_t *p;
<span class="lineNum"> 1696 </span> : uint8_t *q;
<span class="lineNum"> 1697 </span> : uint8_t *r;
<span class="lineNum"> 1698 </span> : uint8_t *s;
<span class="lineNum"> 1699 </span> : uint8_t *t;
<span class="lineNum"> 1700 </span> : uint8_t *u;
<span class="lineNum"> 1701 </span> : int d, l;
<span class="lineNum"> 1702 </span> : size_t i;
<span class="lineNum"> 1703 </span> :
<span class="lineNum"> 1704 </span> : l = nd - 1;
<span class="lineNum"> 1705 </span> : p = v[nd];
<span class="lineNum"> 1706 </span> : q = v[nd + 1];
<span class="lineNum"> 1707 </span> : r = v[nd + 2];
<span class="lineNum"> 1708 </span> : s = v[nd + 3];
<span class="lineNum"> 1709 </span> : t = v[nd + 4];
<span class="lineNum"> 1710 </span> : u = v[nd + 5];
<span class="lineNum"> 1711 </span> :
<span class="lineNum"> 1712 </span> : /* special case with only one data disk */
<span class="lineNum"> 1713 </span> : if (l == 0) {
<span class="lineNum"> 1714 </span> : for (i = 0; i &lt; 6; ++i)
<span class="lineNum"> 1715 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 1716 </span> : return;
<span class="lineNum"> 1717 </span> : }
<span class="lineNum"> 1718 </span> :
<span class="lineNum"> 1719 </span> : raid_sse_begin();
<span class="lineNum"> 1720 </span> :
<span class="lineNum"> 1721 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 1722 </span> : asm volatile (&quot;movdqa %0,%%xmm14&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 1723 </span> : asm volatile (&quot;movdqa %0,%%xmm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1724 </span> :
<span class="lineNum"> 1725 </span> : for (i = 0; i &lt; size; i += 16) {
<span class="lineNum"> 1726 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1727 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 1728 </span> :
<span class="lineNum"> 1729 </span> : asm volatile (&quot;movdqa %xmm10,%xmm0&quot;);
<span class="lineNum"> 1730 </span> : asm volatile (&quot;movdqa %xmm10,%xmm1&quot;);
<span class="lineNum"> 1731 </span> :
<span class="lineNum"> 1732 </span> : asm volatile (&quot;movdqa %xmm10,%xmm11&quot;);
<span class="lineNum"> 1733 </span> : asm volatile (&quot;psrlw $4,%xmm11&quot;);
<span class="lineNum"> 1734 </span> : asm volatile (&quot;pand %xmm15,%xmm10&quot;);
<span class="lineNum"> 1735 </span> : asm volatile (&quot;pand %xmm15,%xmm11&quot;);
<span class="lineNum"> 1736 </span> :
<span class="lineNum"> 1737 </span> : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 1738 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 1739 </span> : asm volatile (&quot;pshufb %xmm10,%xmm2&quot;);
<span class="lineNum"> 1740 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1741 </span> : asm volatile (&quot;pxor %xmm13,%xmm2&quot;);
<span class="lineNum"> 1742 </span> :
<span class="lineNum"> 1743 </span> : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));
<span class="lineNum"> 1744 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));
<span class="lineNum"> 1745 </span> : asm volatile (&quot;pshufb %xmm10,%xmm3&quot;);
<span class="lineNum"> 1746 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1747 </span> : asm volatile (&quot;pxor %xmm13,%xmm3&quot;);
<span class="lineNum"> 1748 </span> :
<span class="lineNum"> 1749 </span> : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (gfgenpshufb[l][2][0][0]));
<span class="lineNum"> 1750 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][2][1][0]));
<span class="lineNum"> 1751 </span> : asm volatile (&quot;pshufb %xmm10,%xmm4&quot;);
<span class="lineNum"> 1752 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1753 </span> : asm volatile (&quot;pxor %xmm13,%xmm4&quot;);
<span class="lineNum"> 1754 </span> :
<span class="lineNum"> 1755 </span> : asm volatile (&quot;movdqa %0,%%xmm5&quot; : : &quot;m&quot; (gfgenpshufb[l][3][0][0]));
<span class="lineNum"> 1756 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[l][3][1][0]));
<span class="lineNum"> 1757 </span> : asm volatile (&quot;pshufb %xmm10,%xmm5&quot;);
<span class="lineNum"> 1758 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1759 </span> : asm volatile (&quot;pxor %xmm13,%xmm5&quot;);
<span class="lineNum"> 1760 </span> :
<span class="lineNum"> 1761 </span> : /* intermediate disks */
<span class="lineNum"> 1762 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 1763 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 1764 </span> :
<span class="lineNum"> 1765 </span> : asm volatile (&quot;pxor %xmm11,%xmm11&quot;);
<span class="lineNum"> 1766 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm11&quot;);
<span class="lineNum"> 1767 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 1768 </span> : asm volatile (&quot;pand %xmm14,%xmm11&quot;);
<span class="lineNum"> 1769 </span> : asm volatile (&quot;pxor %xmm11,%xmm1&quot;);
<span class="lineNum"> 1770 </span> :
<span class="lineNum"> 1771 </span> : asm volatile (&quot;pxor %xmm10,%xmm0&quot;);
<span class="lineNum"> 1772 </span> : asm volatile (&quot;pxor %xmm10,%xmm1&quot;);
<span class="lineNum"> 1773 </span> :
<span class="lineNum"> 1774 </span> : asm volatile (&quot;movdqa %xmm10,%xmm11&quot;);
<span class="lineNum"> 1775 </span> : asm volatile (&quot;psrlw $4,%xmm11&quot;);
<span class="lineNum"> 1776 </span> : asm volatile (&quot;pand %xmm15,%xmm10&quot;);
<span class="lineNum"> 1777 </span> : asm volatile (&quot;pand %xmm15,%xmm11&quot;);
<span class="lineNum"> 1778 </span> :
<span class="lineNum"> 1779 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 1780 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 1781 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1782 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1783 </span> : asm volatile (&quot;pxor %xmm12,%xmm2&quot;);
<span class="lineNum"> 1784 </span> : asm volatile (&quot;pxor %xmm13,%xmm2&quot;);
<span class="lineNum"> 1785 </span> :
<span class="lineNum"> 1786 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));
<span class="lineNum"> 1787 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));
<span class="lineNum"> 1788 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1789 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1790 </span> : asm volatile (&quot;pxor %xmm12,%xmm3&quot;);
<span class="lineNum"> 1791 </span> : asm volatile (&quot;pxor %xmm13,%xmm3&quot;);
<span class="lineNum"> 1792 </span> :
<span class="lineNum"> 1793 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][2][0][0]));
<span class="lineNum"> 1794 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][2][1][0]));
<span class="lineNum"> 1795 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1796 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1797 </span> : asm volatile (&quot;pxor %xmm12,%xmm4&quot;);
<span class="lineNum"> 1798 </span> : asm volatile (&quot;pxor %xmm13,%xmm4&quot;);
<span class="lineNum"> 1799 </span> :
<span class="lineNum"> 1800 </span> : asm volatile (&quot;movdqa %0,%%xmm12&quot; : : &quot;m&quot; (gfgenpshufb[d][3][0][0]));
<span class="lineNum"> 1801 </span> : asm volatile (&quot;movdqa %0,%%xmm13&quot; : : &quot;m&quot; (gfgenpshufb[d][3][1][0]));
<span class="lineNum"> 1802 </span> : asm volatile (&quot;pshufb %xmm10,%xmm12&quot;);
<span class="lineNum"> 1803 </span> : asm volatile (&quot;pshufb %xmm11,%xmm13&quot;);
<span class="lineNum"> 1804 </span> : asm volatile (&quot;pxor %xmm12,%xmm5&quot;);
<span class="lineNum"> 1805 </span> : asm volatile (&quot;pxor %xmm13,%xmm5&quot;);
<span class="lineNum"> 1806 </span> : }
<span class="lineNum"> 1807 </span> :
<span class="lineNum"> 1808 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1809 </span> : asm volatile (&quot;movdqa %0,%%xmm10&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 1810 </span> :
<span class="lineNum"> 1811 </span> : asm volatile (&quot;pxor %xmm11,%xmm11&quot;);
<span class="lineNum"> 1812 </span> : asm volatile (&quot;pcmpgtb %xmm1,%xmm11&quot;);
<span class="lineNum"> 1813 </span> : asm volatile (&quot;paddb %xmm1,%xmm1&quot;);
<span class="lineNum"> 1814 </span> : asm volatile (&quot;pand %xmm14,%xmm11&quot;);
<span class="lineNum"> 1815 </span> : asm volatile (&quot;pxor %xmm11,%xmm1&quot;);
<span class="lineNum"> 1816 </span> :
<span class="lineNum"> 1817 </span> : asm volatile (&quot;pxor %xmm10,%xmm0&quot;);
<span class="lineNum"> 1818 </span> : asm volatile (&quot;pxor %xmm10,%xmm1&quot;);
<span class="lineNum"> 1819 </span> : asm volatile (&quot;pxor %xmm10,%xmm2&quot;);
<span class="lineNum"> 1820 </span> : asm volatile (&quot;pxor %xmm10,%xmm3&quot;);
<span class="lineNum"> 1821 </span> : asm volatile (&quot;pxor %xmm10,%xmm4&quot;);
<span class="lineNum"> 1822 </span> : asm volatile (&quot;pxor %xmm10,%xmm5&quot;);
<span class="lineNum"> 1823 </span> :
<span class="lineNum"> 1824 </span> : asm volatile (&quot;movntdq %%xmm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 1825 </span> : asm volatile (&quot;movntdq %%xmm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 1826 </span> : asm volatile (&quot;movntdq %%xmm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 1827 </span> : asm volatile (&quot;movntdq %%xmm3,%0&quot; : &quot;=m&quot; (s[i]));
<span class="lineNum"> 1828 </span> : asm volatile (&quot;movntdq %%xmm4,%0&quot; : &quot;=m&quot; (t[i]));
<span class="lineNum"> 1829 </span> : asm volatile (&quot;movntdq %%xmm5,%0&quot; : &quot;=m&quot; (u[i]));
<span class="lineNum"> 1830 </span> : }
<span class="lineNum"> 1831 </span> :
<span class="lineNum"> 1832 </span> : raid_sse_end();
<span class="lineNum"> 1833 </span> : }
<span class="lineNum"> 1834 </span> : #endif
<span class="lineNum"> 1835 </span> :
<span class="lineNum"> 1836 </span> : #if defined(CONFIG_X86_64) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 1837 </span> : /*
<span class="lineNum"> 1838 </span> : * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation
<span class="lineNum"> 1839 </span> : *
<span class="lineNum"> 1840 </span> : * Note that it uses 16 registers, meaning that x64 is required.
<span class="lineNum"> 1841 </span> : */
<span class="lineNum"> 1842 </span> : void raid_gen6_avx2ext(int nd, size_t size, void **vv)
<span class="lineNum"> 1843 </span> : {
<span class="lineNum"> 1844 </span> : uint8_t **v = (uint8_t **)vv;
<span class="lineNum"> 1845 </span> : uint8_t *p;
<span class="lineNum"> 1846 </span> : uint8_t *q;
<span class="lineNum"> 1847 </span> : uint8_t *r;
<span class="lineNum"> 1848 </span> : uint8_t *s;
<span class="lineNum"> 1849 </span> : uint8_t *t;
<span class="lineNum"> 1850 </span> : uint8_t *u;
<span class="lineNum"> 1851 </span> : int d, l;
<span class="lineNum"> 1852 </span> : size_t i;
<span class="lineNum"> 1853 </span> :
<span class="lineNum"> 1854 </span> : l = nd - 1;
<span class="lineNum"> 1855 </span> : p = v[nd];
<span class="lineNum"> 1856 </span> : q = v[nd + 1];
<span class="lineNum"> 1857 </span> : r = v[nd + 2];
<span class="lineNum"> 1858 </span> : s = v[nd + 3];
<span class="lineNum"> 1859 </span> : t = v[nd + 4];
<span class="lineNum"> 1860 </span> : u = v[nd + 5];
<span class="lineNum"> 1861 </span> :
<span class="lineNum"> 1862 </span> : /* special case with only one data disk */
<span class="lineNum"> 1863 </span> : if (l == 0) {
<span class="lineNum"> 1864 </span> : for (i = 0; i &lt; 6; ++i)
<span class="lineNum"> 1865 </span> : memcpy(v[1 + i], v[0], size);
<span class="lineNum"> 1866 </span> : return;
<span class="lineNum"> 1867 </span> : }
<span class="lineNum"> 1868 </span> :
<span class="lineNum"> 1869 </span> : raid_avx_begin();
<span class="lineNum"> 1870 </span> :
<span class="lineNum"> 1871 </span> : /* generic case with at least two data disks */
<span class="lineNum"> 1872 </span> : asm volatile (&quot;vpxor %ymm8,%ymm8,%ymm8&quot;);
<span class="lineNum"> 1873 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm14&quot; : : &quot;m&quot; (gfconst16.poly[0]));
<span class="lineNum"> 1874 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm15&quot; : : &quot;m&quot; (gfconst16.low4[0]));
<span class="lineNum"> 1875 </span> :
<span class="lineNum"> 1876 </span> : for (i = 0; i &lt; size; i += 32) {
<span class="lineNum"> 1877 </span> : /* last disk without the by two multiplication */
<span class="lineNum"> 1878 </span> : asm volatile (&quot;vmovdqa %0,%%ymm10&quot; : : &quot;m&quot; (v[l][i]));
<span class="lineNum"> 1879 </span> :
<span class="lineNum"> 1880 </span> : asm volatile (&quot;vmovdqa %ymm10,%ymm0&quot;);
<span class="lineNum"> 1881 </span> : asm volatile (&quot;vmovdqa %ymm10,%ymm1&quot;);
<span class="lineNum"> 1882 </span> :
<span class="lineNum"> 1883 </span> : asm volatile (&quot;vpsrlw $4,%ymm10,%ymm11&quot;);
<span class="lineNum"> 1884 </span> : asm volatile (&quot;vpand %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1885 </span> : asm volatile (&quot;vpand %ymm15,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1886 </span> :
<span class="lineNum"> 1887 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfgenpshufb[l][0][0][0]));
<span class="lineNum"> 1888 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][0][1][0]));
<span class="lineNum"> 1889 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1890 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1891 </span> : asm volatile (&quot;vpxor %ymm13,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1892 </span> :
<span class="lineNum"> 1893 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfgenpshufb[l][1][0][0]));
<span class="lineNum"> 1894 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][1][1][0]));
<span class="lineNum"> 1895 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1896 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1897 </span> : asm volatile (&quot;vpxor %ymm13,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1898 </span> :
<span class="lineNum"> 1899 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm4&quot; : : &quot;m&quot; (gfgenpshufb[l][2][0][0]));
<span class="lineNum"> 1900 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][2][1][0]));
<span class="lineNum"> 1901 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1902 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1903 </span> : asm volatile (&quot;vpxor %ymm13,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1904 </span> :
<span class="lineNum"> 1905 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm5&quot; : : &quot;m&quot; (gfgenpshufb[l][3][0][0]));
<span class="lineNum"> 1906 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[l][3][1][0]));
<span class="lineNum"> 1907 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1908 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1909 </span> : asm volatile (&quot;vpxor %ymm13,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1910 </span> :
<span class="lineNum"> 1911 </span> : /* intermediate disks */
<span class="lineNum"> 1912 </span> : for (d = l - 1; d &gt; 0; --d) {
<span class="lineNum"> 1913 </span> : asm volatile (&quot;vmovdqa %0,%%ymm10&quot; : : &quot;m&quot; (v[d][i]));
<span class="lineNum"> 1914 </span> :
<span class="lineNum"> 1915 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm8,%ymm11&quot;);
<span class="lineNum"> 1916 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1917 </span> : asm volatile (&quot;vpand %ymm14,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1918 </span> : asm volatile (&quot;vpxor %ymm11,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1919 </span> :
<span class="lineNum"> 1920 </span> : asm volatile (&quot;vpxor %ymm10,%ymm0,%ymm0&quot;);
<span class="lineNum"> 1921 </span> : asm volatile (&quot;vpxor %ymm10,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1922 </span> :
<span class="lineNum"> 1923 </span> : asm volatile (&quot;vpsrlw $4,%ymm10,%ymm11&quot;);
<span class="lineNum"> 1924 </span> : asm volatile (&quot;vpand %ymm15,%ymm10,%ymm10&quot;);
<span class="lineNum"> 1925 </span> : asm volatile (&quot;vpand %ymm15,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1926 </span> :
<span class="lineNum"> 1927 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][0][0][0]));
<span class="lineNum"> 1928 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][0][1][0]));
<span class="lineNum"> 1929 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1930 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1931 </span> : asm volatile (&quot;vpxor %ymm12,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1932 </span> : asm volatile (&quot;vpxor %ymm13,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1933 </span> :
<span class="lineNum"> 1934 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][1][0][0]));
<span class="lineNum"> 1935 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][1][1][0]));
<span class="lineNum"> 1936 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1937 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1938 </span> : asm volatile (&quot;vpxor %ymm12,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1939 </span> : asm volatile (&quot;vpxor %ymm13,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1940 </span> :
<span class="lineNum"> 1941 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][2][0][0]));
<span class="lineNum"> 1942 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][2][1][0]));
<span class="lineNum"> 1943 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1944 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1945 </span> : asm volatile (&quot;vpxor %ymm12,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1946 </span> : asm volatile (&quot;vpxor %ymm13,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1947 </span> :
<span class="lineNum"> 1948 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm12&quot; : : &quot;m&quot; (gfgenpshufb[d][3][0][0]));
<span class="lineNum"> 1949 </span> : asm volatile (&quot;vbroadcasti128 %0,%%ymm13&quot; : : &quot;m&quot; (gfgenpshufb[d][3][1][0]));
<span class="lineNum"> 1950 </span> : asm volatile (&quot;vpshufb %ymm10,%ymm12,%ymm12&quot;);
<span class="lineNum"> 1951 </span> : asm volatile (&quot;vpshufb %ymm11,%ymm13,%ymm13&quot;);
<span class="lineNum"> 1952 </span> : asm volatile (&quot;vpxor %ymm12,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1953 </span> : asm volatile (&quot;vpxor %ymm13,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1954 </span> : }
<span class="lineNum"> 1955 </span> :
<span class="lineNum"> 1956 </span> : /* first disk with all coefficients at 1 */
<span class="lineNum"> 1957 </span> : asm volatile (&quot;vmovdqa %0,%%ymm10&quot; : : &quot;m&quot; (v[0][i]));
<span class="lineNum"> 1958 </span> :
<span class="lineNum"> 1959 </span> : asm volatile (&quot;vpcmpgtb %ymm1,%ymm8,%ymm11&quot;);
<span class="lineNum"> 1960 </span> : asm volatile (&quot;vpaddb %ymm1,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1961 </span> : asm volatile (&quot;vpand %ymm14,%ymm11,%ymm11&quot;);
<span class="lineNum"> 1962 </span> : asm volatile (&quot;vpxor %ymm11,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1963 </span> :
<span class="lineNum"> 1964 </span> : asm volatile (&quot;vpxor %ymm10,%ymm0,%ymm0&quot;);
<span class="lineNum"> 1965 </span> : asm volatile (&quot;vpxor %ymm10,%ymm1,%ymm1&quot;);
<span class="lineNum"> 1966 </span> : asm volatile (&quot;vpxor %ymm10,%ymm2,%ymm2&quot;);
<span class="lineNum"> 1967 </span> : asm volatile (&quot;vpxor %ymm10,%ymm3,%ymm3&quot;);
<span class="lineNum"> 1968 </span> : asm volatile (&quot;vpxor %ymm10,%ymm4,%ymm4&quot;);
<span class="lineNum"> 1969 </span> : asm volatile (&quot;vpxor %ymm10,%ymm5,%ymm5&quot;);
<span class="lineNum"> 1970 </span> :
<span class="lineNum"> 1971 </span> : asm volatile (&quot;vmovntdq %%ymm0,%0&quot; : &quot;=m&quot; (p[i]));
<span class="lineNum"> 1972 </span> : asm volatile (&quot;vmovntdq %%ymm1,%0&quot; : &quot;=m&quot; (q[i]));
<span class="lineNum"> 1973 </span> : asm volatile (&quot;vmovntdq %%ymm2,%0&quot; : &quot;=m&quot; (r[i]));
<span class="lineNum"> 1974 </span> : asm volatile (&quot;vmovntdq %%ymm3,%0&quot; : &quot;=m&quot; (s[i]));
<span class="lineNum"> 1975 </span> : asm volatile (&quot;vmovntdq %%ymm4,%0&quot; : &quot;=m&quot; (t[i]));
<span class="lineNum"> 1976 </span> : asm volatile (&quot;vmovntdq %%ymm5,%0&quot; : &quot;=m&quot; (u[i]));
<span class="lineNum"> 1977 </span> : }
<span class="lineNum"> 1978 </span> :
<span class="lineNum"> 1979 </span> : raid_avx_end();
<span class="lineNum"> 1980 </span> : }
<span class="lineNum"> 1981 </span> : #endif
<span class="lineNum"> 1982 </span> :
<span class="lineNum"> 1983 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 1984 </span> : /*
<a name="1985"><span class="lineNum"> 1985 </span> : * RAID recovering for one disk SSSE3 implementation</a>
<span class="lineNum"> 1986 </span> : */
<span class="lineNum"> 1987 </span><span class="lineCov"> 111395 : void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span>
<span class="lineNum"> 1988 </span> : {
<span class="lineNum"> 1989 </span><span class="lineCov"> 111395 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 1990 </span> : uint8_t *p;
<span class="lineNum"> 1991 </span> : uint8_t *pa;
<span class="lineNum"> 1992 </span> : uint8_t G;
<span class="lineNum"> 1993 </span> : uint8_t V;
<span class="lineNum"> 1994 </span> : size_t i;
<span class="lineNum"> 1995 </span> :
<span class="lineNum"> 1996 </span> : (void)nr; /* unused, it's always 1 */
<span class="lineNum"> 1997 </span> :
<span class="lineNum"> 1998 </span> : /* if it's RAID5 uses the faster function */
<span class="lineNum"> 1999 </span><span class="lineCov"> 111395 : if (ip[0] == 0) {</span>
<span class="lineNum"> 2000 </span><span class="lineCov"> 101506 : raid_rec1of1(id, nd, size, vv);</span>
<span class="lineNum"> 2001 </span><span class="lineCov"> 212901 : return;</span>
<span class="lineNum"> 2002 </span> : }
<span class="lineNum"> 2003 </span> :
<span class="lineNum"> 2004 </span> : /* setup the coefficients matrix */
<span class="lineNum"> 2005 </span><span class="lineCov"> 19778 : G = A(ip[0], id[0]);</span>
<span class="lineNum"> 2006 </span> :
<span class="lineNum"> 2007 </span> : /* invert it to solve the system of linear equations */
<span class="lineNum"> 2008 </span><span class="lineCov"> 19778 : V = inv(G);</span>
<span class="lineNum"> 2009 </span> :
<span class="lineNum"> 2010 </span> : /* compute delta parity */
<span class="lineNum"> 2011 </span><span class="lineCov"> 9889 : raid_delta_gen(1, id, ip, nd, size, vv);</span>
<span class="lineNum"> 2012 </span> :
<span class="lineNum"> 2013 </span><span class="lineCov"> 9889 : p = v[nd + ip[0]];</span>
<span class="lineNum"> 2014 </span><span class="lineCov"> 9889 : pa = v[id[0]];</span>
<span class="lineNum"> 2015 </span> :
<span class="lineNum"> 2016 </span> : raid_sse_begin();
<span class="lineNum"> 2017 </span> :
<span class="lineNum"> 2018 </span><span class="lineCov"> 9889 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 2019 </span><span class="lineCov"> 9889 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (gfmulpshufb[V][0][0]));</span>
<span class="lineNum"> 2020 </span><span class="lineCov"> 9889 : asm volatile (&quot;movdqa %0,%%xmm5&quot; : : &quot;m&quot; (gfmulpshufb[V][1][0]));</span>
<span class="lineNum"> 2021 </span> :
<span class="lineNum"> 2022 </span><span class="lineCov"> 1552673 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 2023 </span><span class="lineCov"> 1542784 : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (p[i]));</span>
<span class="lineNum"> 2024 </span><span class="lineCov"> 1542784 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (pa[i]));</span>
<span class="lineNum"> 2025 </span><span class="lineCov"> 1542784 : asm volatile (&quot;movdqa %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 2026 </span><span class="lineCov"> 1542784 : asm volatile (&quot;movdqa %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 2027 </span><span class="lineCov"> 1542784 : asm volatile (&quot;pxor %xmm0,%xmm1&quot;);</span>
<span class="lineNum"> 2028 </span><span class="lineCov"> 1542784 : asm volatile (&quot;movdqa %xmm1,%xmm0&quot;);</span>
<span class="lineNum"> 2029 </span><span class="lineCov"> 1542784 : asm volatile (&quot;psrlw $4,%xmm1&quot;);</span>
<span class="lineNum"> 2030 </span><span class="lineCov"> 1542784 : asm volatile (&quot;pand %xmm7,%xmm0&quot;);</span>
<span class="lineNum"> 2031 </span><span class="lineCov"> 1542784 : asm volatile (&quot;pand %xmm7,%xmm1&quot;);</span>
<span class="lineNum"> 2032 </span><span class="lineCov"> 1542784 : asm volatile (&quot;pshufb %xmm0,%xmm2&quot;);</span>
<span class="lineNum"> 2033 </span><span class="lineCov"> 1542784 : asm volatile (&quot;pshufb %xmm1,%xmm3&quot;);</span>
<span class="lineNum"> 2034 </span><span class="lineCov"> 1542784 : asm volatile (&quot;pxor %xmm3,%xmm2&quot;);</span>
<span class="lineNum"> 2035 </span><span class="lineCov"> 1542784 : asm volatile (&quot;movdqa %%xmm2,%0&quot; : &quot;=m&quot; (pa[i]));</span>
<span class="lineNum"> 2036 </span> : }
<span class="lineNum"> 2037 </span> :
<span class="lineNum"> 2038 </span> : raid_sse_end();
<span class="lineNum"> 2039 </span> : }
<span class="lineNum"> 2040 </span> : #endif
<span class="lineNum"> 2041 </span> :
<span class="lineNum"> 2042 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 2043 </span> : /*
<a name="2044"><span class="lineNum"> 2044 </span> : * RAID recovering for two disks SSSE3 implementation</a>
<span class="lineNum"> 2045 </span> : */
<span class="lineNum"> 2046 </span><span class="lineCov"> 44038 : void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span>
<span class="lineNum"> 2047 </span> : {
<span class="lineNum"> 2048 </span><span class="lineCov"> 44038 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 2049 </span><span class="lineCov"> 44038 : const int N = 2;</span>
<span class="lineNum"> 2050 </span><span class="lineCov"> 44038 : uint8_t *p[N];</span>
<span class="lineNum"> 2051 </span><span class="lineCov"> 44038 : uint8_t *pa[N];</span>
<span class="lineNum"> 2052 </span><span class="lineCov"> 44038 : uint8_t G[N * N];</span>
<span class="lineNum"> 2053 </span><span class="lineCov"> 44038 : uint8_t V[N * N];</span>
<span class="lineNum"> 2054 </span> : size_t i;
<span class="lineNum"> 2055 </span> : int j, k;
<span class="lineNum"> 2056 </span> :
<span class="lineNum"> 2057 </span> : (void)nr; /* unused, it's always 2 */
<span class="lineNum"> 2058 </span> :
<span class="lineNum"> 2059 </span> : /* setup the coefficients matrix */
<span class="lineNum"> 2060 </span><span class="lineCov"> 132114 : for (j = 0; j &lt; N; ++j)</span>
<span class="lineNum"> 2061 </span><span class="lineCov"> 264228 : for (k = 0; k &lt; N; ++k)</span>
<span class="lineNum"> 2062 </span><span class="lineCov"> 352304 : G[j * N + k] = A(ip[j], id[k]);</span>
<span class="lineNum"> 2063 </span> :
<span class="lineNum"> 2064 </span> : /* invert it to solve the system of linear equations */
<span class="lineNum"> 2065 </span><span class="lineCov"> 44038 : raid_invert(G, V, N);</span>
<span class="lineNum"> 2066 </span> :
<span class="lineNum"> 2067 </span> : /* compute delta parity */
<span class="lineNum"> 2068 </span><span class="lineCov"> 44038 : raid_delta_gen(N, id, ip, nd, size, vv);</span>
<span class="lineNum"> 2069 </span> :
<span class="lineNum"> 2070 </span><span class="lineCov"> 132114 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2071 </span><span class="lineCov"> 88076 : p[j] = v[nd + ip[j]];</span>
<span class="lineNum"> 2072 </span><span class="lineCov"> 88076 : pa[j] = v[id[j]];</span>
<span class="lineNum"> 2073 </span> : }
<span class="lineNum"> 2074 </span> :
<span class="lineNum"> 2075 </span> : raid_sse_begin();
<span class="lineNum"> 2076 </span> :
<span class="lineNum"> 2077 </span><span class="lineCov"> 44038 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 2078 </span> :
<span class="lineNum"> 2079 </span><span class="lineCov"> 3588806 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 2080 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (p[0][i]));</span>
<span class="lineNum"> 2081 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (pa[0][i]));</span>
<span class="lineNum"> 2082 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (p[1][i]));</span>
<span class="lineNum"> 2083 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (pa[1][i]));</span>
<span class="lineNum"> 2084 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm2,%xmm0&quot;);</span>
<span class="lineNum"> 2085 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm3,%xmm1&quot;);</span>
<span class="lineNum"> 2086 </span> :
<span class="lineNum"> 2087 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm6,%xmm6&quot;);</span>
<span class="lineNum"> 2088 </span> :
<span class="lineNum"> 2089 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfmulpshufb[V[0]][0][0]));</span>
<span class="lineNum"> 2090 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfmulpshufb[V[0]][1][0]));</span>
<span class="lineNum"> 2091 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm0,%xmm4&quot;);</span>
<span class="lineNum"> 2092 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm0,%xmm5&quot;);</span>
<span class="lineNum"> 2093 </span><span class="lineCov"> 3544768 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 2094 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 2095 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 2096 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 2097 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 2098 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm2,%xmm6&quot;);</span>
<span class="lineNum"> 2099 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm3,%xmm6&quot;);</span>
<span class="lineNum"> 2100 </span> :
<span class="lineNum"> 2101 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfmulpshufb[V[1]][0][0]));</span>
<span class="lineNum"> 2102 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfmulpshufb[V[1]][1][0]));</span>
<span class="lineNum"> 2103 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm1,%xmm4&quot;);</span>
<span class="lineNum"> 2104 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm1,%xmm5&quot;);</span>
<span class="lineNum"> 2105 </span><span class="lineCov"> 3544768 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 2106 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 2107 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 2108 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 2109 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 2110 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm2,%xmm6&quot;);</span>
<span class="lineNum"> 2111 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm3,%xmm6&quot;);</span>
<span class="lineNum"> 2112 </span> :
<span class="lineNum"> 2113 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %%xmm6,%0&quot; : &quot;=m&quot; (pa[0][i]));</span>
<span class="lineNum"> 2114 </span> :
<span class="lineNum"> 2115 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm6,%xmm6&quot;);</span>
<span class="lineNum"> 2116 </span> :
<span class="lineNum"> 2117 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfmulpshufb[V[2]][0][0]));</span>
<span class="lineNum"> 2118 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfmulpshufb[V[2]][1][0]));</span>
<span class="lineNum"> 2119 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm0,%xmm4&quot;);</span>
<span class="lineNum"> 2120 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm0,%xmm5&quot;);</span>
<span class="lineNum"> 2121 </span><span class="lineCov"> 3544768 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 2122 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 2123 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 2124 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 2125 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 2126 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm2,%xmm6&quot;);</span>
<span class="lineNum"> 2127 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm3,%xmm6&quot;);</span>
<span class="lineNum"> 2128 </span> :
<span class="lineNum"> 2129 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfmulpshufb[V[3]][0][0]));</span>
<span class="lineNum"> 2130 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfmulpshufb[V[3]][1][0]));</span>
<span class="lineNum"> 2131 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm1,%xmm4&quot;);</span>
<span class="lineNum"> 2132 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %xmm1,%xmm5&quot;);</span>
<span class="lineNum"> 2133 </span><span class="lineCov"> 3544768 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 2134 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 2135 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 2136 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 2137 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pshufb %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 2138 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm2,%xmm6&quot;);</span>
<span class="lineNum"> 2139 </span><span class="lineCov"> 3544768 : asm volatile (&quot;pxor %xmm3,%xmm6&quot;);</span>
<span class="lineNum"> 2140 </span> :
<span class="lineNum"> 2141 </span><span class="lineCov"> 3544768 : asm volatile (&quot;movdqa %%xmm6,%0&quot; : &quot;=m&quot; (pa[1][i]));</span>
<span class="lineNum"> 2142 </span> : }
<span class="lineNum"> 2143 </span> :
<span class="lineNum"> 2144 </span> : raid_sse_end();
<span class="lineNum"> 2145 </span><span class="lineCov"> 44038 : }</span>
<span class="lineNum"> 2146 </span> : #endif
<span class="lineNum"> 2147 </span> :
<span class="lineNum"> 2148 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_SSSE3)
<span class="lineNum"> 2149 </span> : /*
<a name="2150"><span class="lineNum"> 2150 </span> : * RAID recovering SSSE3 implementation</a>
<span class="lineNum"> 2151 </span> : */
<span class="lineNum"> 2152 </span><span class="lineCov"> 79288 : void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span>
<span class="lineNum"> 2153 </span> : {
<span class="lineNum"> 2154 </span><span class="lineCov"> 79288 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 2155 </span><span class="lineCov"> 79288 : int N = nr;</span>
<span class="lineNum"> 2156 </span> : uint8_t *p[RAID_PARITY_MAX];
<span class="lineNum"> 2157 </span> : uint8_t *pa[RAID_PARITY_MAX];
<span class="lineNum"> 2158 </span> : uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
<span class="lineNum"> 2159 </span> : uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
<span class="lineNum"> 2160 </span> : uint8_t buffer[RAID_PARITY_MAX*16+16];
<span class="lineNum"> 2161 </span><span class="lineCov"> 79288 : uint8_t *pd = __align_ptr(buffer, 16);</span>
<span class="lineNum"> 2162 </span> : size_t i;
<span class="lineNum"> 2163 </span> : int j, k;
<span class="lineNum"> 2164 </span> :
<span class="lineNum"> 2165 </span> : /* setup the coefficients matrix */
<span class="lineNum"> 2166 </span><span class="lineCov"> 435102 : for (j = 0; j &lt; N; ++j)</span>
<span class="lineNum"> 2167 </span><span class="lineCov"> 2046374 : for (k = 0; k &lt; N; ++k)</span>
<span class="lineNum"> 2168 </span><span class="lineCov"> 3381120 : G[j * N + k] = A(ip[j], id[k]);</span>
<span class="lineNum"> 2169 </span> :
<span class="lineNum"> 2170 </span> : /* invert it to solve the system of linear equations */
<span class="lineNum"> 2171 </span><span class="lineCov"> 79288 : raid_invert(G, V, N);</span>
<span class="lineNum"> 2172 </span> :
<span class="lineNum"> 2173 </span> : /* compute delta parity */
<span class="lineNum"> 2174 </span><span class="lineCov"> 79288 : raid_delta_gen(N, id, ip, nd, size, vv);</span>
<span class="lineNum"> 2175 </span> :
<span class="lineNum"> 2176 </span><span class="lineCov"> 435102 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2177 </span><span class="lineCov"> 355814 : p[j] = v[nd + ip[j]];</span>
<span class="lineNum"> 2178 </span><span class="lineCov"> 355814 : pa[j] = v[id[j]];</span>
<span class="lineNum"> 2179 </span> : }
<span class="lineNum"> 2180 </span> :
<span class="lineNum"> 2181 </span> : raid_sse_begin();
<span class="lineNum"> 2182 </span> :
<span class="lineNum"> 2183 </span><span class="lineCov"> 79288 : asm volatile (&quot;movdqa %0,%%xmm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 2184 </span> :
<span class="lineNum"> 2185 </span><span class="lineCov"> 5869832 : for (i = 0; i &lt; size; i += 16) {</span>
<span class="lineNum"> 2186 </span> : /* delta */
<span class="lineNum"> 2187 </span><span class="lineCov"> 32115408 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2188 </span><span class="lineCov"> 26324864 : asm volatile (&quot;movdqa %0,%%xmm0&quot; : : &quot;m&quot; (p[j][i]));</span>
<span class="lineNum"> 2189 </span><span class="lineCov"> 26324864 : asm volatile (&quot;movdqa %0,%%xmm1&quot; : : &quot;m&quot; (pa[j][i]));</span>
<span class="lineNum"> 2190 </span><span class="lineCov"> 26324864 : asm volatile (&quot;pxor %xmm1,%xmm0&quot;);</span>
<span class="lineNum"> 2191 </span><span class="lineCov"> 26324864 : asm volatile (&quot;movdqa %%xmm0,%0&quot; : &quot;=m&quot; (pd[j*16]));</span>
<span class="lineNum"> 2192 </span> : }
<span class="lineNum"> 2193 </span> :
<span class="lineNum"> 2194 </span> : /* reconstruct */
<span class="lineNum"> 2195 </span><span class="lineCov"> 32115408 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2196 </span><span class="lineCov"> 26324864 : asm volatile (&quot;pxor %xmm0,%xmm0&quot;);</span>
<span class="lineNum"> 2197 </span><span class="lineCov"> 26324864 : asm volatile (&quot;pxor %xmm1,%xmm1&quot;);</span>
<span class="lineNum"> 2198 </span> :
<span class="lineNum"> 2199 </span><span class="lineCov"> 153207872 : for (k = 0; k &lt; N; ++k) {</span>
<span class="lineNum"> 2200 </span><span class="lineCov"> 126883008 : uint8_t m = V[j * N + k];</span>
<span class="lineNum"> 2201 </span> :
<span class="lineNum"> 2202 </span><span class="lineCov"> 126883008 : asm volatile (&quot;movdqa %0,%%xmm2&quot; : : &quot;m&quot; (gfmulpshufb[m][0][0]));</span>
<span class="lineNum"> 2203 </span><span class="lineCov"> 126883008 : asm volatile (&quot;movdqa %0,%%xmm3&quot; : : &quot;m&quot; (gfmulpshufb[m][1][0]));</span>
<span class="lineNum"> 2204 </span><span class="lineCov"> 126883008 : asm volatile (&quot;movdqa %0,%%xmm4&quot; : : &quot;m&quot; (pd[k*16]));</span>
<span class="lineNum"> 2205 </span><span class="lineCov"> 126883008 : asm volatile (&quot;movdqa %xmm4,%xmm5&quot;);</span>
<span class="lineNum"> 2206 </span><span class="lineCov"> 126883008 : asm volatile (&quot;psrlw $4,%xmm5&quot;);</span>
<span class="lineNum"> 2207 </span><span class="lineCov"> 126883008 : asm volatile (&quot;pand %xmm7,%xmm4&quot;);</span>
<span class="lineNum"> 2208 </span><span class="lineCov"> 126883008 : asm volatile (&quot;pand %xmm7,%xmm5&quot;);</span>
<span class="lineNum"> 2209 </span><span class="lineCov"> 126883008 : asm volatile (&quot;pshufb %xmm4,%xmm2&quot;);</span>
<span class="lineNum"> 2210 </span><span class="lineCov"> 126883008 : asm volatile (&quot;pshufb %xmm5,%xmm3&quot;);</span>
<span class="lineNum"> 2211 </span><span class="lineCov"> 126883008 : asm volatile (&quot;pxor %xmm2,%xmm0&quot;);</span>
<span class="lineNum"> 2212 </span><span class="lineCov"> 126883008 : asm volatile (&quot;pxor %xmm3,%xmm1&quot;);</span>
<span class="lineNum"> 2213 </span> : }
<span class="lineNum"> 2214 </span> :
<span class="lineNum"> 2215 </span><span class="lineCov"> 26324864 : asm volatile (&quot;pxor %xmm1,%xmm0&quot;);</span>
<span class="lineNum"> 2216 </span><span class="lineCov"> 26324864 : asm volatile (&quot;movdqa %%xmm0,%0&quot; : &quot;=m&quot; (pa[j][i]));</span>
<span class="lineNum"> 2217 </span> : }
<span class="lineNum"> 2218 </span> : }
<span class="lineNum"> 2219 </span> :
<span class="lineNum"> 2220 </span> : raid_sse_end();
<span class="lineNum"> 2221 </span><span class="lineCov"> 79288 : }</span>
<span class="lineNum"> 2222 </span> : #endif
<span class="lineNum"> 2223 </span> :
<span class="lineNum"> 2224 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 2225 </span> : /*
<a name="2226"><span class="lineNum"> 2226 </span> : * RAID recovering for one disk AVX2 implementation</a>
<span class="lineNum"> 2227 </span> : */
<span class="lineNum"> 2228 </span><span class="lineCov"> 126 : void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span>
<span class="lineNum"> 2229 </span> : {
<span class="lineNum"> 2230 </span><span class="lineCov"> 126 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 2231 </span> : uint8_t *p;
<span class="lineNum"> 2232 </span> : uint8_t *pa;
<span class="lineNum"> 2233 </span> : uint8_t G;
<span class="lineNum"> 2234 </span> : uint8_t V;
<span class="lineNum"> 2235 </span> : size_t i;
<span class="lineNum"> 2236 </span> :
<span class="lineNum"> 2237 </span> : (void)nr; /* unused, it's always 1 */
<span class="lineNum"> 2238 </span> :
<span class="lineNum"> 2239 </span> : /* if it's RAID5 uses the faster function */
<span class="lineNum"> 2240 </span><span class="lineCov"> 126 : if (ip[0] == 0) {</span>
<span class="lineNum"> 2241 </span><span class="lineCov"> 30 : raid_rec1of1(id, nd, size, vv);</span>
<span class="lineNum"> 2242 </span><span class="lineCov"> 156 : return;</span>
<span class="lineNum"> 2243 </span> : }
<span class="lineNum"> 2244 </span> :
<span class="lineNum"> 2245 </span> : /* setup the coefficients matrix */
<span class="lineNum"> 2246 </span><span class="lineCov"> 192 : G = A(ip[0], id[0]);</span>
<span class="lineNum"> 2247 </span> :
<span class="lineNum"> 2248 </span> : /* invert it to solve the system of linear equations */
<span class="lineNum"> 2249 </span><span class="lineCov"> 192 : V = inv(G);</span>
<span class="lineNum"> 2250 </span> :
<span class="lineNum"> 2251 </span> : /* compute delta parity */
<span class="lineNum"> 2252 </span><span class="lineCov"> 96 : raid_delta_gen(1, id, ip, nd, size, vv);</span>
<span class="lineNum"> 2253 </span> :
<span class="lineNum"> 2254 </span><span class="lineCov"> 96 : p = v[nd + ip[0]];</span>
<span class="lineNum"> 2255 </span><span class="lineCov"> 96 : pa = v[id[0]];</span>
<span class="lineNum"> 2256 </span> :
<span class="lineNum"> 2257 </span> : raid_avx_begin();
<span class="lineNum"> 2258 </span> :
<span class="lineNum"> 2259 </span><span class="lineCov"> 96 : asm volatile (&quot;vbroadcasti128 %0,%%ymm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 2260 </span><span class="lineCov"> 96 : asm volatile (&quot;vbroadcasti128 %0,%%ymm4&quot; : : &quot;m&quot; (gfmulpshufb[V][0][0]));</span>
<span class="lineNum"> 2261 </span><span class="lineCov"> 96 : asm volatile (&quot;vbroadcasti128 %0,%%ymm5&quot; : : &quot;m&quot; (gfmulpshufb[V][1][0]));</span>
<span class="lineNum"> 2262 </span> :
<span class="lineNum"> 2263 </span><span class="lineCov"> 66816 : for (i = 0; i &lt; size; i += 32) {</span>
<span class="lineNum"> 2264 </span><span class="lineCov"> 66720 : asm volatile (&quot;vmovdqa %0,%%ymm0&quot; : : &quot;m&quot; (p[i]));</span>
<span class="lineNum"> 2265 </span><span class="lineCov"> 66720 : asm volatile (&quot;vmovdqa %0,%%ymm1&quot; : : &quot;m&quot; (pa[i]));</span>
<span class="lineNum"> 2266 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpxor %ymm1,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2267 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpsrlw $4,%ymm0,%ymm1&quot;);</span>
<span class="lineNum"> 2268 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpand %ymm7,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2269 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpand %ymm7,%ymm1,%ymm1&quot;);</span>
<span class="lineNum"> 2270 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpshufb %ymm0,%ymm4,%ymm2&quot;);</span>
<span class="lineNum"> 2271 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpshufb %ymm1,%ymm5,%ymm3&quot;);</span>
<span class="lineNum"> 2272 </span><span class="lineCov"> 66720 : asm volatile (&quot;vpxor %ymm3,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 2273 </span><span class="lineCov"> 66720 : asm volatile (&quot;vmovdqa %%ymm2,%0&quot; : &quot;=m&quot; (pa[i]));</span>
<span class="lineNum"> 2274 </span> : }
<span class="lineNum"> 2275 </span> :
<span class="lineNum"> 2276 </span> : raid_avx_end();
<span class="lineNum"> 2277 </span> : }
<span class="lineNum"> 2278 </span> : #endif
<span class="lineNum"> 2279 </span> :
<span class="lineNum"> 2280 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 2281 </span> : /*
<a name="2282"><span class="lineNum"> 2282 </span> : * RAID recovering for two disks AVX2 implementation</a>
<span class="lineNum"> 2283 </span> : */
<span class="lineNum"> 2284 </span><span class="lineCov"> 1206 : void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span>
<span class="lineNum"> 2285 </span> : {
<span class="lineNum"> 2286 </span><span class="lineCov"> 1206 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 2287 </span><span class="lineCov"> 1206 : const int N = 2;</span>
<span class="lineNum"> 2288 </span><span class="lineCov"> 1206 : uint8_t *p[N];</span>
<span class="lineNum"> 2289 </span><span class="lineCov"> 1206 : uint8_t *pa[N];</span>
<span class="lineNum"> 2290 </span><span class="lineCov"> 1206 : uint8_t G[N * N];</span>
<span class="lineNum"> 2291 </span><span class="lineCov"> 1206 : uint8_t V[N * N];</span>
<span class="lineNum"> 2292 </span> : size_t i;
<span class="lineNum"> 2293 </span> : int j, k;
<span class="lineNum"> 2294 </span> :
<span class="lineNum"> 2295 </span> : (void)nr; /* unused, it's always 2 */
<span class="lineNum"> 2296 </span> :
<span class="lineNum"> 2297 </span> : /* setup the coefficients matrix */
<span class="lineNum"> 2298 </span><span class="lineCov"> 3618 : for (j = 0; j &lt; N; ++j)</span>
<span class="lineNum"> 2299 </span><span class="lineCov"> 7236 : for (k = 0; k &lt; N; ++k)</span>
<span class="lineNum"> 2300 </span><span class="lineCov"> 9648 : G[j * N + k] = A(ip[j], id[k]);</span>
<span class="lineNum"> 2301 </span> :
<span class="lineNum"> 2302 </span> : /* invert it to solve the system of linear equations */
<span class="lineNum"> 2303 </span><span class="lineCov"> 1206 : raid_invert(G, V, N);</span>
<span class="lineNum"> 2304 </span> :
<span class="lineNum"> 2305 </span> : /* compute delta parity */
<span class="lineNum"> 2306 </span><span class="lineCov"> 1206 : raid_delta_gen(N, id, ip, nd, size, vv);</span>
<span class="lineNum"> 2307 </span> :
<span class="lineNum"> 2308 </span><span class="lineCov"> 3618 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2309 </span><span class="lineCov"> 2412 : p[j] = v[nd + ip[j]];</span>
<span class="lineNum"> 2310 </span><span class="lineCov"> 2412 : pa[j] = v[id[j]];</span>
<span class="lineNum"> 2311 </span> : }
<span class="lineNum"> 2312 </span> :
<span class="lineNum"> 2313 </span> : raid_avx_begin();
<span class="lineNum"> 2314 </span> :
<span class="lineNum"> 2315 </span><span class="lineCov"> 1206 : asm volatile (&quot;vbroadcasti128 %0,%%ymm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 2316 </span> :
<span class="lineNum"> 2317 </span><span class="lineCov"> 77526 : for (i = 0; i &lt; size; i += 32) {</span>
<span class="lineNum"> 2318 </span><span class="lineCov"> 76320 : asm volatile (&quot;vmovdqa %0,%%ymm0&quot; : : &quot;m&quot; (p[0][i]));</span>
<span class="lineNum"> 2319 </span><span class="lineCov"> 76320 : asm volatile (&quot;vmovdqa %0,%%ymm2&quot; : : &quot;m&quot; (pa[0][i]));</span>
<span class="lineNum"> 2320 </span><span class="lineCov"> 76320 : asm volatile (&quot;vmovdqa %0,%%ymm1&quot; : : &quot;m&quot; (p[1][i]));</span>
<span class="lineNum"> 2321 </span><span class="lineCov"> 76320 : asm volatile (&quot;vmovdqa %0,%%ymm3&quot; : : &quot;m&quot; (pa[1][i]));</span>
<span class="lineNum"> 2322 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm2,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2323 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm3,%ymm1,%ymm1&quot;);</span>
<span class="lineNum"> 2324 </span> :
<span class="lineNum"> 2325 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm6,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2326 </span> :
<span class="lineNum"> 2327 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfmulpshufb[V[0]][0][0]));</span>
<span class="lineNum"> 2328 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfmulpshufb[V[0]][1][0]));</span>
<span class="lineNum"> 2329 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpsrlw $4,%ymm0,%ymm5&quot;);</span>
<span class="lineNum"> 2330 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm0,%ymm4&quot;);</span>
<span class="lineNum"> 2331 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);</span>
<span class="lineNum"> 2332 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 2333 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 2334 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm2,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2335 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm3,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2336 </span> :
<span class="lineNum"> 2337 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfmulpshufb[V[1]][0][0]));</span>
<span class="lineNum"> 2338 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfmulpshufb[V[1]][1][0]));</span>
<span class="lineNum"> 2339 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpsrlw $4,%ymm1,%ymm5&quot;);</span>
<span class="lineNum"> 2340 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm1,%ymm4&quot;);</span>
<span class="lineNum"> 2341 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);</span>
<span class="lineNum"> 2342 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 2343 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 2344 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm2,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2345 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm3,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2346 </span> :
<span class="lineNum"> 2347 </span><span class="lineCov"> 76320 : asm volatile (&quot;vmovdqa %%ymm6,%0&quot; : &quot;=m&quot; (pa[0][i]));</span>
<span class="lineNum"> 2348 </span> :
<span class="lineNum"> 2349 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm6,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2350 </span> :
<span class="lineNum"> 2351 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfmulpshufb[V[2]][0][0]));</span>
<span class="lineNum"> 2352 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfmulpshufb[V[2]][1][0]));</span>
<span class="lineNum"> 2353 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpsrlw $4,%ymm0,%ymm5&quot;);</span>
<span class="lineNum"> 2354 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm0,%ymm4&quot;);</span>
<span class="lineNum"> 2355 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);</span>
<span class="lineNum"> 2356 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 2357 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 2358 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm2,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2359 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm3,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2360 </span> :
<span class="lineNum"> 2361 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfmulpshufb[V[3]][0][0]));</span>
<span class="lineNum"> 2362 </span><span class="lineCov"> 76320 : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfmulpshufb[V[3]][1][0]));</span>
<span class="lineNum"> 2363 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpsrlw $4,%ymm1,%ymm5&quot;);</span>
<span class="lineNum"> 2364 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm1,%ymm4&quot;);</span>
<span class="lineNum"> 2365 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);</span>
<span class="lineNum"> 2366 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 2367 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpshufb %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 2368 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm2,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2369 </span><span class="lineCov"> 76320 : asm volatile (&quot;vpxor %ymm3,%ymm6,%ymm6&quot;);</span>
<span class="lineNum"> 2370 </span> :
<span class="lineNum"> 2371 </span><span class="lineCov"> 76320 : asm volatile (&quot;vmovdqa %%ymm6,%0&quot; : &quot;=m&quot; (pa[1][i]));</span>
<span class="lineNum"> 2372 </span> : }
<span class="lineNum"> 2373 </span> :
<span class="lineNum"> 2374 </span> : raid_avx_end();
<span class="lineNum"> 2375 </span><span class="lineCov"> 1206 : }</span>
<span class="lineNum"> 2376 </span> : #endif
<span class="lineNum"> 2377 </span> :
<span class="lineNum"> 2378 </span> : #if defined(CONFIG_X86) &amp;&amp; defined(CONFIG_AVX2)
<span class="lineNum"> 2379 </span> : /*
<a name="2380"><span class="lineNum"> 2380 </span> : * RAID recovering AVX2 implementation</a>
<span class="lineNum"> 2381 </span> : */
<span class="lineNum"> 2382 </span><span class="lineCov"> 17765 : void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)</span>
<span class="lineNum"> 2383 </span> : {
<span class="lineNum"> 2384 </span><span class="lineCov"> 17765 : uint8_t **v = (uint8_t **)vv;</span>
<span class="lineNum"> 2385 </span><span class="lineCov"> 17765 : int N = nr;</span>
<span class="lineNum"> 2386 </span> : uint8_t *p[RAID_PARITY_MAX];
<span class="lineNum"> 2387 </span> : uint8_t *pa[RAID_PARITY_MAX];
<span class="lineNum"> 2388 </span> : uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
<span class="lineNum"> 2389 </span> : uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
<span class="lineNum"> 2390 </span> : uint8_t buffer[RAID_PARITY_MAX*32+32];
<span class="lineNum"> 2391 </span><span class="lineCov"> 17765 : uint8_t *pd = __align_ptr(buffer, 32);</span>
<span class="lineNum"> 2392 </span> : size_t i;
<span class="lineNum"> 2393 </span> : int j, k;
<span class="lineNum"> 2394 </span> :
<span class="lineNum"> 2395 </span> : /* setup the coefficients matrix */
<span class="lineNum"> 2396 </span><span class="lineCov"> 90821 : for (j = 0; j &lt; N; ++j)</span>
<span class="lineNum"> 2397 </span><span class="lineCov"> 386396 : for (k = 0; k &lt; N; ++k)</span>
<span class="lineNum"> 2398 </span><span class="lineCov"> 626680 : G[j * N + k] = A(ip[j], id[k]);</span>
<span class="lineNum"> 2399 </span> :
<span class="lineNum"> 2400 </span> : /* invert it to solve the system of linear equations */
<span class="lineNum"> 2401 </span><span class="lineCov"> 17765 : raid_invert(G, V, N);</span>
<span class="lineNum"> 2402 </span> :
<span class="lineNum"> 2403 </span> : /* compute delta parity */
<span class="lineNum"> 2404 </span><span class="lineCov"> 17765 : raid_delta_gen(N, id, ip, nd, size, vv);</span>
<span class="lineNum"> 2405 </span> :
<span class="lineNum"> 2406 </span><span class="lineCov"> 90821 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2407 </span><span class="lineCov"> 73056 : p[j] = v[nd + ip[j]];</span>
<span class="lineNum"> 2408 </span><span class="lineCov"> 73056 : pa[j] = v[id[j]];</span>
<span class="lineNum"> 2409 </span> : }
<span class="lineNum"> 2410 </span> :
<span class="lineNum"> 2411 </span> : raid_avx_begin();
<span class="lineNum"> 2412 </span> :
<span class="lineNum"> 2413 </span><span class="lineCov"> 17765 : asm volatile (&quot;vbroadcasti128 %0,%%ymm7&quot; : : &quot;m&quot; (gfconst16.low4[0]));</span>
<span class="lineNum"> 2414 </span> :
<span class="lineNum"> 2415 </span><span class="lineCov"> 423213 : for (i = 0; i &lt; size; i += 32) {</span>
<span class="lineNum"> 2416 </span> : /* delta */
<span class="lineNum"> 2417 </span><span class="lineCov"> 2174152 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2418 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vmovdqa %0,%%ymm0&quot; : : &quot;m&quot; (p[j][i]));</span>
<span class="lineNum"> 2419 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vmovdqa %0,%%ymm1&quot; : : &quot;m&quot; (pa[j][i]));</span>
<span class="lineNum"> 2420 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vpxor %ymm1,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2421 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vmovdqa %%ymm0,%0&quot; : &quot;=m&quot; (pd[j*32]));</span>
<span class="lineNum"> 2422 </span> : }
<span class="lineNum"> 2423 </span> :
<span class="lineNum"> 2424 </span> : /* reconstruct */
<span class="lineNum"> 2425 </span><span class="lineCov"> 2174152 : for (j = 0; j &lt; N; ++j) {</span>
<span class="lineNum"> 2426 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vpxor %ymm0,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2427 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vpxor %ymm1,%ymm1,%ymm1&quot;);</span>
<span class="lineNum"> 2428 </span> :
<span class="lineNum"> 2429 </span><span class="lineCov"> 9930976 : for (k = 0; k &lt; N; ++k) {</span>
<span class="lineNum"> 2430 </span><span class="lineCov"> 8162272 : uint8_t m = V[j * N + k];</span>
<span class="lineNum"> 2431 </span> :
<span class="lineNum"> 2432 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vbroadcasti128 %0,%%ymm2&quot; : : &quot;m&quot; (gfmulpshufb[m][0][0]));</span>
<span class="lineNum"> 2433 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vbroadcasti128 %0,%%ymm3&quot; : : &quot;m&quot; (gfmulpshufb[m][1][0]));</span>
<span class="lineNum"> 2434 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vmovdqa %0,%%ymm4&quot; : : &quot;m&quot; (pd[k*32]));</span>
<span class="lineNum"> 2435 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpsrlw $4,%ymm4,%ymm5&quot;);</span>
<span class="lineNum"> 2436 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpand %ymm7,%ymm4,%ymm4&quot;);</span>
<span class="lineNum"> 2437 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpand %ymm7,%ymm5,%ymm5&quot;);</span>
<span class="lineNum"> 2438 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpshufb %ymm4,%ymm2,%ymm2&quot;);</span>
<span class="lineNum"> 2439 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpshufb %ymm5,%ymm3,%ymm3&quot;);</span>
<span class="lineNum"> 2440 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpxor %ymm2,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2441 </span><span class="lineCov"> 8162272 : asm volatile (&quot;vpxor %ymm3,%ymm1,%ymm1&quot;);</span>
<span class="lineNum"> 2442 </span> : }
<span class="lineNum"> 2443 </span> :
<span class="lineNum"> 2444 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vpxor %ymm1,%ymm0,%ymm0&quot;);</span>
<span class="lineNum"> 2445 </span><span class="lineCov"> 1768704 : asm volatile (&quot;vmovdqa %%ymm0,%0&quot; : &quot;=m&quot; (pa[j][i]));</span>
<span class="lineNum"> 2446 </span> : }
<span class="lineNum"> 2447 </span> : }
<span class="lineNum"> 2448 </span> :
<span class="lineNum"> 2449 </span> : raid_avx_end();
<span class="lineNum"> 2450 </span><span class="lineCov"> 17765 : }</span>
<span class="lineNum"> 2451 </span> : #endif
<span class="lineNum"> 2452 </span> :
</pre>
</td>
</tr>
</table>
<br>
<table width="100%" border=0 cellspacing=0 cellpadding=0>
<tr><td class="ruler"><img src="../glass.png" width=3 height=3 alt=""></td></tr>
<tr><td class="versionInfo">Generated by: <a href="http://ltp.sourceforge.net/coverage/lcov.php" target="_parent">LCOV version 1.10</a></td></tr>
</table>
<br>
</body>
</html>