diff --git a/cmdline/speed.c b/cmdline/speed.c
index 54b99846..841bb40c 100644
--- a/cmdline/speed.c
+++ b/cmdline/speed.c
@@ -1226,10 +1226,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	printf("%8s", "neon32");
 #endif
 #ifdef CONFIG_X86
-	if (raid_cpu_has_ssse3())
+	if (raid_cpu_has_ssse3()) {
 		printf("%8s", "ssse3");
-	if (raid_cpu_has_avx2())
+#ifdef CONFIG_X86_64
+		printf("%8s", "ssse3e");
+#endif
+	}
+	if (raid_cpu_has_avx2()) {
 		printf("%8s", "avx2");
+#ifdef CONFIG_X86_64
+		printf("%8s", "avx2e");
+#endif
+	}
 #ifdef CONFIG_X86_64
 	if (raid_cpu_has_avx512bw())
 		printf("%8s", "avx512");
@@ -1257,7 +1265,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
+		/* raid_rec1of1() internally calls raid_gen() */
 		raid_gen_force(1, raid_gen1_neon);
 		raid_rec1_neon(1, id, ip, nd, size, v);
 	} SPEED_STOP
@@ -1267,7 +1275,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
+		/* raid_rec1of1() internally calls raid_gen() */
 		raid_gen_force(1, raid_gen1_neon32);
 		raid_rec1_neon32(1, id, ip, nd, size, v);
 	} SPEED_STOP
@@ -1279,13 +1287,24 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
+			/* raid_rec1of1() internally calls raid_gen() */
 			raid_gen_force(1, raid_gen1_sse2);
 			raid_rec1_ssse3(1, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* raid_rec1of1() internally calls raid_gen() */
+			raid_gen_force(1, raid_gen1_sse2);
+			raid_rec1_ssse3ext(1, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
@@ -1298,9 +1317,22 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/*
+			 * raid_rec1_avx2ext() uses raid_rec1of1() for P,
+			 * which internally calls raid_gen().
+			 */
+			raid_gen_force(1, raid_gen1_avx2);
+			raid_rec1_avx2ext(1, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
+			/* raid_rec1of1() internally calls raid_gen() */
 			raid_gen_force(1, raid_gen1_avx512bw);
 			raid_rec1_avx512bw(1, id, ip, nd, size, v);
 		} SPEED_STOP
@@ -1310,7 +1342,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
+			/* raid_rec1of1() internally calls raid_gen() */
 			raid_gen_force(1, raid_gen1_avx2);
 			raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v);
 		} SPEED_STOP
@@ -1320,7 +1352,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
+			/* raid_rec1of1() internally calls raid_gen() */
 			raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
 			raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v);
 		} SPEED_STOP
@@ -1349,8 +1381,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(1, raid_gen1_neon);
 		/* +1 to avoid GEN1 optimized case */
 		raid_rec1_neon(1, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1360,8 +1390,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(1, raid_gen1_neon32);
 		/* +1 to avoid GEN1 optimized case */
 		raid_rec1_neon32(1, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1381,6 +1409,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_sse2);
+			/* +1 to avoid GEN1 optimized case */
+			raid_rec1_ssse3ext(1, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
@@ -1394,11 +1434,20 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
-	if (raid_cpu_has_avx512bw()) {
+	if (raid_cpu_has_avx2()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-			raid_gen_force(1, raid_gen1_avx512bw);
+			raid_gen_force(1, raid_gen1_avx2);
 			/* +1 to avoid GEN1 optimized case */
+			raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
+			/* +1 to avoid RAID5 optimized case */
 			raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
@@ -1407,9 +1456,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(1, raid_gen1_avx2);
-			/* +1 to avoid GEN1 optimized case */
+			/* +1 to avoid RAID5 optimized case */
 			raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
@@ -1418,9 +1465,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
-			/* +1 to avoid GEN1 optimized case */
+			/* +1 to avoid RAID5 optimized case */
 			raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
@@ -1431,6 +1476,118 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 	printf("\n");
 
+	/* REC1OF3 */
+	printf("%8s", "rec1of3");
+	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1));
+	fflush(stdout);
+
+	SPEED_START {
+		/* ensure to use same hardware in the delta step */
+		raid_gen_force(1, sizeof(void*) == 8 ? raid_gen1_int64 : raid_gen1_int32);
+		/* +2 to select R instead of Q */
+		raid_rec1_int8(1, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+
+#ifdef CONFIG_NEON
+	SPEED_START {
+		/* +2 to select R instead of Q */
+		raid_rec1_neon(1, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+#ifdef CONFIG_NEON32
+	SPEED_START {
+		/* +2 to select R instead of Q */
+		raid_rec1_neon32(1, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+
+#ifdef CONFIG_X86
+	if (raid_cpu_has_ssse3()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_sse2);
+			/* +2 to select R instead of Q */
+			raid_rec1_ssse3(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_sse2);
+			/* +2 to select R instead of Q */
+			raid_rec1_ssse3ext(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
+	}
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_avx2);
+			/* +2 to select R instead of Q */
+			raid_rec1_avx2(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_avx2);
+			/* +2 to select R instead of Q */
+			raid_rec1_avx2ext(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
+			/* +2 to select R instead of Q */
+			raid_rec1_avx512bw(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx2gfni()) {
+		SPEED_START {
+			/* +2 to select R instead of Q */
+			raid_rec1_avx2gfni_raid(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512gfni()) {
+		SPEED_START {
+			/* +2 to select R instead of Q */
+			raid_rec1_avx512gfni_raid(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#endif
+#endif
+	printf("\n");
+
 	/* REC2OF2 */
 	printf("%8s", "rec2of2");
 	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
@@ -1447,8 +1604,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(2, raid_gen2_neon_raid);
 		raid_rec2_neon(2, id, ip, nd, size, v);
 	} SPEED_STOP
 
@@ -1457,8 +1612,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(2, raid_gen2_neon32_raid);
 		raid_rec2_neon32(2, id, ip, nd, size, v);
 	} SPEED_STOP
 
@@ -1470,25 +1623,28 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_sse2ext_raid);
-#else
 			raid_gen_force(2, raid_gen2_sse2_raid);
-#endif
 			raid_rec2_ssse3(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2ext_raid);
+			raid_rec2_ssse3ext(2, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_avx2ext_raid);
-#else
 			raid_gen_force(2, raid_gen2_avx2_raid);
-#endif
 			raid_rec2_avx2(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1496,10 +1652,18 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
-	if (raid_cpu_has_avx512bw()) {
+	if (raid_cpu_has_avx2()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-			raid_gen_force(2, raid_gen2_avx512bw);
+			raid_gen_force(2, raid_gen2_avx2ext_raid);
+			raid_rec2_avx2ext(2, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
 			raid_rec2_avx512bw(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1508,8 +1672,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(2, raid_gen2_avx2gfni_raid);
 			raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1518,8 +1680,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(2, raid_gen2_avx512gfni_raid);
 			raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1547,8 +1707,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(2, raid_gen2_neon_raid);
 		/* +1 to avoid GEN2 optimized case */
 		raid_rec2_neon(2, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1558,8 +1716,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(2, raid_gen2_neon32_raid);
 		/* +1 to avoid GEN2 optimized case */
 		raid_rec2_neon32(2, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1572,26 +1728,30 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_sse2ext_raid);
-#else
 			raid_gen_force(2, raid_gen2_sse2_raid);
-#endif
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_ssse3(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2ext_raid);
+			/* +1 to avoid GEN2 optimized case */
+			raid_rec2_ssse3ext(2, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_avx2ext_raid);
-#else
 			raid_gen_force(2, raid_gen2_avx2_raid);
-#endif
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx2(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1600,10 +1760,19 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
-	if (raid_cpu_has_avx512bw()) {
+	if (raid_cpu_has_avx2()) {
 		SPEED_START {
 			/* ensure to use same hardware in the delta step */
-			raid_gen_force(2, raid_gen2_avx512bw);
+			raid_gen_force(2, raid_gen2_avx2ext_raid);
+			/* +1 to avoid GEN2 optimized case */
+			raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1613,8 +1782,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(2, raid_gen2_avx2gfni_raid);
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1624,8 +1791,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(2, raid_gen2_avx512gfni_raid);
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1637,6 +1802,118 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 	printf("\n");
 
+	/* REC2OF4 */
+	printf("%8s", "rec2of4");
+	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
+	fflush(stdout);
+
+	SPEED_START {
+		/* ensure to use same hardware in the delta step */
+		raid_gen_force(2, sizeof(void*) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
+		/* +2 to select R,S instead of Q,R */
+		raid_rec2_int8(2, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+
+#ifdef CONFIG_NEON
+	SPEED_START {
+		/* +2 to select R,S instead of Q,R */
+		raid_rec2_neon(2, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+#ifdef CONFIG_NEON32
+	SPEED_START {
+		/* +2 to select R,S instead of Q,R */
+		raid_rec2_neon32(2, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+
+#ifdef CONFIG_X86
+	if (raid_cpu_has_ssse3()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_ssse3(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2ext_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_ssse3ext(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
+	}
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_avx2_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx2(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_avx2ext_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx2ext(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx512bw(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx2gfni()) {
+		SPEED_START {
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx2gfni_raid(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512gfni()) {
+		SPEED_START {
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx512gfni_raid(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#endif
+#endif
+	printf("\n");
+
 	printf("%8s", "rec3of3");
 	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3));
 	fflush(stdout);
@@ -1652,9 +1929,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(3, raid_gen3_neon_raid);
-		raid_recX_neon(3, id, ip, nd, size, v);
+		raid_rec3_neon(3, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1662,9 +1937,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(3, raid_gen3_neon32_raid);
-		raid_recX_neon32(3, id, ip, nd, size, v);
+		raid_rec3_neon32(3, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1674,39 +1947,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(3, raid_gen3_ssse3ext_raid);
-#else
-			raid_gen_force(3, raid_gen3_ssse3_raid);
-#endif
-			raid_recX_ssse3(3, id, ip, nd, size, v);
+			raid_rec3_ssse3(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec3_ssse3ext(3, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(3, raid_gen3_avx2ext_raid);
-#else
-			raid_gen_force(3, raid_gen3_ssse3_raid);
-#endif
-			raid_recX_avx2(3, id, ip, nd, size, v);
+			raid_rec3_avx2(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec3_avx2ext(3, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(3, raid_gen3_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(3, id, ip, nd, size, v);
+			raid_rec3_avx512bw(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1714,9 +1989,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(3, raid_gen3_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
+			raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1724,9 +1997,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(3, raid_gen3_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
+			raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1751,9 +2022,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(4, raid_gen4_neon_raid);
-		raid_recX_neon(4, id, ip, nd, size, v);
+		raid_rec4_neon(4, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1761,9 +2030,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(4, raid_gen4_neon32_raid);
-		raid_recX_neon32(4, id, ip, nd, size, v);
+		raid_rec4_neon32(4, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1773,40 +2040,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(4, raid_gen4_ssse3ext_raid);
-#else
-
-			raid_gen_force(4, raid_gen4_ssse3_raid);
-#endif
-			raid_recX_ssse3(4, id, ip, nd, size, v);
+			raid_rec4_ssse3(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec4_ssse3ext(4, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(4, raid_gen4_avx2ext_raid);
-#else
-			raid_gen_force(4, raid_gen4_ssse3_raid);
-#endif
-			raid_recX_avx2(4, id, ip, nd, size, v);
+			raid_rec4_avx2(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec4_avx2ext(4, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(4, raid_gen4_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(4, id, ip, nd, size, v);
+			raid_rec4_avx512bw(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1814,9 +2082,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(4, raid_gen4_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
+			raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1824,9 +2090,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(4, raid_gen4_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
+			raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1851,9 +2115,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(5, raid_gen5_neon_raid);
-		raid_recX_neon(5, id, ip, nd, size, v);
+		raid_rec5_neon(5, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1861,9 +2123,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(5, raid_gen5_neon32_raid);
-		raid_recX_neon32(5, id, ip, nd, size, v);
+		raid_rec5_neon32(5, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1873,39 +2133,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(5, raid_gen5_ssse3ext_raid);
-#else
-			raid_gen_force(5, raid_gen5_ssse3_raid);
-#endif
-			raid_recX_ssse3(5, id, ip, nd, size, v);
+			raid_rec5_ssse3(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec5_ssse3ext(5, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(5, raid_gen5_avx2ext_raid);
-#else
-			raid_gen_force(5, raid_gen5_ssse3_raid);
-#endif
-			raid_recX_avx2(5, id, ip, nd, size, v);
+			raid_rec5_avx2(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec5_avx2ext(5, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(5, raid_gen5_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(5, id, ip, nd, size, v);
+			raid_rec5_avx512bw(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1913,9 +2175,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(5, raid_gen5_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
+			raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1923,9 +2183,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(5, raid_gen5_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
+			raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1950,9 +2208,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(6, raid_gen6_neon_raid);
-		raid_recX_neon(6, id, ip, nd, size, v);
+		raid_rec6_neon(6, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1960,9 +2216,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		/* ensure to use same hardware in the delta step */
-		raid_gen_force(6, raid_gen6_neon32_raid);
-		raid_recX_neon32(6, id, ip, nd, size, v);
+		raid_rec6_neon32(6, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1972,39 +2226,41 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(6, raid_gen6_ssse3ext_raid);
-#else
-			raid_gen_force(6, raid_gen6_ssse3_raid);
-#endif
-			raid_recX_ssse3(6, id, ip, nd, size, v);
+			raid_rec6_ssse3(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec6_ssse3ext(6, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-#ifdef CONFIG_X86_64
-			raid_gen_force(6, raid_gen6_avx2ext_raid);
-#else
-			raid_gen_force(6, raid_gen6_ssse3_raid);
-#endif
-			raid_recX_avx2(6, id, ip, nd, size, v);
+			raid_rec6_avx2(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec6_avx2ext(6, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(6, raid_gen6_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(6, id, ip, nd, size, v);
+			raid_rec6_avx512bw(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -2012,9 +2268,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(6, raid_gen6_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
+			raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -2022,9 +2276,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			/* ensure to use same hardware in the delta step */
-			raid_gen_force(6, raid_gen6_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
+			raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
diff --git a/raid/avx2.c b/raid/avx2.c
index 3271724b..dfe230ac 100644
--- a/raid/avx2.c
+++ b/raid/avx2.c
@@ -6,7 +6,6 @@
 #include "cpu.h"
 
 #ifdef CONFIG_X86
-
 /*
  * GEN1 (RAID5 with xor) AVX2 implementation
  */
@@ -106,16 +105,6 @@ static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, i
 	raid_avx_end();
 }
 
-void raid_gen2_avx2_raid(int nd, size_t size, void **vv)
-{
-	raid_gen2_avx2_gen(nd, size, vv, 2);
-}
-
-void raid_gen2_avx2_aes(int nd, size_t size, void **vv)
-{
-	raid_gen2_avx2_gen(nd, size, vv, 3);
-}
-
 #ifdef CONFIG_X86_64
 /*
  * GEN2 Cauchy AVX2 implementation using the extended register set.
@@ -125,8 +114,7 @@ void raid_gen2_avx2_aes(int nd, size_t size, void **vv)
  *
  * Note that it uses 16 registers, meaning that x64 is required.
  */
-static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
-	void **vv, int generator)
+static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size, void **vv, int generator)
 {
 	uint8_t **v = (uint8_t **)vv;
 	uint8_t *p;
@@ -232,16 +220,6 @@ static __always_inline void raid_gen2_avx2ext_gen(int nd, size_t size,
 	raid_avx_end();
 }
 
-void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv)
-{
-	raid_gen2_avx2ext_gen(nd, size, vv, 2);
-}
-
-void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv)
-{
-	raid_gen2_avx2ext_gen(nd, size, vv, 3);
-}
-
 /*
  * GENz (triple parity with powers of 2^-1) AVX2 implementation
  *
@@ -316,9 +294,7 @@ void raid_genz_avx2ext_raid(int nd, size_t size, void **vv)
 
 	raid_avx_end();
 }
-#endif
 
-#ifdef CONFIG_X86_64
 /*
  * GEN3 (triple parity with Cauchy matrix) AVX2 implementation
  *
@@ -574,18 +550,6 @@ static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv
 	raid_avx_end();
 }
 
-void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv)
-{
-	raid_gen3_avx2ext_gen(nd, size, vv, 2);
-}
-
-void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv)
-{
-	raid_gen3_avx2ext_gen(nd, size, vv, 3);
-}
-#endif
-
-#ifdef CONFIG_X86_64
 /*
  * GEN4 (quad parity with Cauchy matrix) AVX2 implementation
  *
@@ -758,18 +722,6 @@ static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv
 	raid_avx_end();
 }
 
-void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv)
-{
-	raid_gen4_avx2ext_gen(nd, size, vv, 2);
-}
-
-void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv)
-{
-	raid_gen4_avx2ext_gen(nd, size, vv, 3);
-}
-#endif
-
-#ifdef CONFIG_X86_64
 /*
  * GENX AVX2EXT implementation
  */
@@ -924,46 +876,13 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
 
 	raid_avx_end();
 }
-#endif
 
-#ifdef CONFIG_X86_64
-/*
- * GEN5 (penta parity with Cauchy matrix) AVX2 implementation
- *
- * Note that it uses 16 registers, meaning that x64 is required.
- */
-void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv)
-{
-	raid_genX_avx2ext(nd, size, vv, 5, 2);
-}
-
-void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv)
-{
-	raid_genX_avx2ext(nd, size, vv, 5, 3);
-}
-#endif
-
-#ifdef CONFIG_X86_64
-/*
- * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation
- *
- * Note that it uses 16 registers, meaning that x64 is required.
- */
-void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv)
-{
-	raid_genX_avx2ext(nd, size, vv, 6, 2);
-}
-
-void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv)
-{
-	raid_genX_avx2ext(nd, size, vv, 6, 3);
-}
 #endif
 
 /*
  * RAID recovering for one disk AVX2 implementation
  */
-void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+static __always_inline void raid_rec1_avx2_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
 	uint8_t **v = (uint8_t **)vv;
 	uint8_t *p;
@@ -974,12 +893,6 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	(void)nr; /* unused, it's always 1 */
 
-	/* if it's RAID5 uses the faster function */
-	if (ip[0] == 0) {
-		raid_rec1of1(id, nd, size, vv);
-		return;
-	}
-
 	/* setup the coefficients matrix */
 	G = A(ip[0], id[0]);
 
@@ -1033,7 +946,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 /*
  * RAID recovering for two disks AVX2 implementation
  */
-void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+static __always_inline void raid_rec2_avx2_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
 	uint8_t **v = (uint8_t **)vv;
 	const int N = 2;
@@ -1064,74 +977,6 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	raid_avx_begin();
 
-#ifdef CONFIG_X86_64
-	asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.low4[0]));
-
-	/* the inverse matrix V[] is constant for the whole recovery. */
-	asm volatile ("vbroadcasti128 %0,%%ymm8" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm9" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
-	asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
-
-	for (i = 0; i < size; i += 32) {
-		/* d0 = p[0] ^ pa[0] */
-		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
-		asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
-		asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
-
-		/* d1 = p[1] ^ pa[1] */
-		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i]));
-		asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[1][i]));
-		asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
-
-		/*
-		 * Split both deltas into low/high nibbles once.
-		 *
-		 * ymm0 = d0 low
-		 * ymm2 = d0 high
-		 * ymm1 = d1 low
-		 * ymm3 = d1 high
-		 */
-		asm volatile ("vpsrlw $4,%ymm0,%ymm2");
-		asm volatile ("vpsrlw $4,%ymm1,%ymm3");
-		asm volatile ("vpand %ymm6,%ymm0,%ymm0");
-		asm volatile ("vpand %ymm6,%ymm2,%ymm2");
-		asm volatile ("vpand %ymm6,%ymm1,%ymm1");
-		asm volatile ("vpand %ymm6,%ymm3,%ymm3");
-
-		/*
-		 * pa[0] = V[0] * d0 ^ V[1] * d1
-		 */
-		asm volatile ("vpshufb %ymm0,%ymm8,%ymm4");
-		asm volatile ("vpshufb %ymm2,%ymm9,%ymm5");
-		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
-
-		asm volatile ("vpshufb %ymm1,%ymm10,%ymm5");
-		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
-		asm volatile ("vpshufb %ymm3,%ymm11,%ymm5");
-		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
-
-		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
-
-		/*
-		 * pa[1] = V[2] * d0 ^ V[3] * d1
-		 */
-		asm volatile ("vpshufb %ymm0,%ymm12,%ymm4");
-		asm volatile ("vpshufb %ymm2,%ymm13,%ymm5");
-		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
-
-		asm volatile ("vpshufb %ymm1,%ymm14,%ymm5");
-		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
-		asm volatile ("vpshufb %ymm3,%ymm15,%ymm5");
-		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
-
-		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[1][i]));
-	}
-#else /* CONFIG_X86_32 */
 	asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
 
 	for (i = 0; i < size; i += 32) {
@@ -1190,25 +1035,424 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 		asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
 	}
-#endif
 
 	raid_avx_end();
 }
 
 /*
- * RAID recovering AVX2 implementation
+ * RAID recovering AVX2 implementation optimized for up to four failures.
+ *
+ * Compute all selected syndromes in one scan of the surviving data, then
+ * retain only their low/high nibbles while reconstructing the missing data.
  */
-void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+static __always_inline void raid_recX_avx2_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
 	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
 	uint8_t *p[RAID_PARITY_MAX];
 	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
 	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
 	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	uint8_t buffer_low[RAID_PARITY_MAX * 32 + 32];
+	uint8_t buffer_high[RAID_PARITY_MAX * 32 + 32];
+	uint8_t *pd_low = __align_ptr(buffer_low, 32);
+	uint8_t *pd_high = __align_ptr(buffer_high, 32);
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > 4);
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, nr);
+
+	/* setup selected parity and destination pointers */
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	/* ip[] is ordered. If P is available, it is always ip[0] */
+	has_p = ip[0] == 0;
+
+	/* build the compact surviving-data list and its syndrome tables */
+	ns = 0;
+	k = 0;
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	/* precompute inverse-matrix multiplication table pointers */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_avx_begin();
+
+	asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 32) {
+		/* ymm0..ymm3 are syndrome accumulators */
+		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
+		if (nr >= 2)
+			asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i]));
+		if (nr >= 3)
+			asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[2][i]));
+		if (nr >= 4)
+			asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[3][i]));
+
+		/* all selected syndromes are accumulated in this single source scan */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (src[s][i]));
+			if (has_p) {
+				asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
+
+				asm volatile ("vpsrlw $4,%ymm4,%ymm5");
+				asm volatile ("vpand %ymm7,%ymm4,%ymm4");
+				asm volatile ("vpand %ymm7,%ymm5,%ymm5");
+			} else {
+				asm volatile ("vpsrlw $4,%ymm4,%ymm5");
+				asm volatile ("vpand %ymm7,%ymm4,%ymm4");
+				asm volatile ("vpand %ymm7,%ymm5,%ymm5");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[0][0]));
+				asm volatile ("vpshufb %ymm4,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[0][16]));
+				asm volatile ("vpshufb %ymm5,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[1][0]));
+				asm volatile ("vpshufb %ymm4,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[1][16]));
+				asm volatile ("vpshufb %ymm5,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[2][0]));
+				asm volatile ("vpshufb %ymm4,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[2][16]));
+				asm volatile ("vpshufb %ymm5,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[3][0]));
+				asm volatile ("vpshufb %ymm4,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm3,%ymm3");
+				asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (t[3][16]));
+				asm volatile ("vpshufb %ymm5,%ymm6,%ymm6");
+				asm volatile ("vpxor %ymm6,%ymm3,%ymm3");
+			}
+		}
+
+		/* preserve only aligned low/high nibbles, never full delta blocks */
+		asm volatile ("vpsrlw $4,%ymm0,%ymm4");
+		asm volatile ("vpand %ymm7,%ymm0,%ymm0");
+		asm volatile ("vpand %ymm7,%ymm4,%ymm4");
+		asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd_low[0]));
+		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[0]));
+
+		if (nr >= 2) {
+			asm volatile ("vpsrlw $4,%ymm1,%ymm4");
+			asm volatile ("vpand %ymm7,%ymm1,%ymm1");
+			asm volatile ("vpand %ymm7,%ymm4,%ymm4");
+			asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd_low[32]));
+			asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[32]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vpsrlw $4,%ymm2,%ymm4");
+			asm volatile ("vpand %ymm7,%ymm2,%ymm2");
+			asm volatile ("vpand %ymm7,%ymm4,%ymm4");
+			asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pd_low[64]));
+			asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[64]));
+		}
+
+		if (nr >= 4) {
+			asm volatile ("vpsrlw $4,%ymm3,%ymm4");
+			asm volatile ("vpand %ymm7,%ymm3,%ymm3");
+			asm volatile ("vpand %ymm7,%ymm4,%ymm4");
+			asm volatile ("vmovdqa %%ymm3,%0" : "=m" (pd_low[96]));
+			asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_high[96]));
+		}
+
+		/* reconstruct every missing block, initializing from coefficient zero */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("vbroadcasti128 %0,%%ymm0" : : "m" (t[0][0]));
+			asm volatile ("vbroadcasti128 %0,%%ymm1" : : "m" (t[0][16]));
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[0]));
+			asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[0]));
+			asm volatile ("vpshufb %ymm4,%ymm0,%ymm0");
+			asm volatile ("vpshufb %ymm5,%ymm1,%ymm1");
+
+			for (k = 1; k < nr; ++k) {
+				asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (t[k][0]));
+				asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (t[k][16]));
+				asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[k * 32]));
+				asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[k * 32]));
+				asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");
+				asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");
+				asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
+			}
+
+			asm volatile ("vpxor %ymm1,%ymm0,%ymm0");
+			asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_avx_end();
+}
+
+/*
+ * RAID recovering AVX2 implementation with all syndromes in memory.
+ *
+ * Compute all selected syndromes in a single pass over the surviving data.
+ * Syndrome accumulators are kept in memory to minimize register pressure.
+ */
+static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	uint8_t buffer_low[RAID_PARITY_MAX * 32 + 32];
+	uint8_t buffer_high[RAID_PARITY_MAX * 32 + 32];
+	uint8_t *pd_low = __align_ptr(buffer_low, 32);
+	uint8_t *pd_high = __align_ptr(buffer_high, 32);
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, nr);
+
+	/* setup selected parity and destination pointers */
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	/* ip[] is ordered. If P is available, it is always ip[0] */
+	has_p = ip[0] == 0;
+
+	/* build the compact surviving-data list and its syndrome tables */
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	/* precompute inverse-matrix multiplication table pointers */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_avx_begin();
+
+	asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 32) {
+		/*
+		 * Initialize all syndrome accumulators in memory from the
+		 * selected stored parity blocks.
+		 *
+		 * pd_low[] temporarily contains the complete raw syndromes.
+		 */
+		for (j = 0; j < nr; ++j) {
+			asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i]));
+			asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd_low[j * 32]));
+		}
+
+		/*
+		 * Single pass over all surviving data.
+		 *
+		 * Every src[s][i] is loaded exactly once.
+		 */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			/* original source */
+			asm volatile ("vmovdqa %0,%%ymm0" : : "m" (src[s][i]));
+
+			/*
+			 * P has coefficient 1.
+			 *
+			 * Update its memory accumulator before destructively
+			 * splitting the source into low/high nibbles.
+			 */
+			if (has_p) {
+				asm volatile ("vpxor %0,%%ymm0,%%ymm4" : : "m" (pd_low[0]));
+				asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pd_low[0]));
+			}
+
+			/* split source into low/high nibbles */
+			asm volatile ("vpsrlw $4,%ymm0,%ymm1");
+			asm volatile ("vpand %ymm7,%ymm0,%ymm0");
+			asm volatile ("vpand %ymm7,%ymm1,%ymm1");
+
+			/*
+			 * Update every non-P syndrome in memory.
+			 *
+			 * ymm0 = source low
+			 * ymm1 = source high
+			 * ymm2 = low table / low result
+			 * ymm3 = high table / high result
+			 */
+			for (j = has_p; j < nr; ++j) {
+				asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (t[j][0]));
+				asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (t[j][16]));
+
+				asm volatile ("vpshufb %ymm0,%ymm2,%ymm2");
+				asm volatile ("vpshufb %ymm1,%ymm3,%ymm3");
+
+				asm volatile ("vpxor %ymm3,%ymm2,%ymm2");
+
+				/*
+				 * Syndrome[j] ^= coefficient * source
+				 *
+				 * Read the accumulator directly as the memory
+				 * operand of VPXOR, then write it back.
+				 */
+				asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (pd_low[j * 32]));
+				asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pd_low[j * 32]));
+			}
+		}
+
+		/*
+		 * All survivor reads for this chunk are now complete.
+		 *
+		 * Convert each raw syndrome once to low/high nibble form.
+		 * pd_low[] is overwritten in place.
+		 */
+		for (k = 0; k < nr; ++k) {
+			asm volatile ("vmovdqa %0,%%ymm0" : : "m" (pd_low[k * 32]));
+			asm volatile ("vpsrlw $4,%ymm0,%ymm1");
+			asm volatile ("vpand %ymm7,%ymm0,%ymm0");
+			asm volatile ("vpand %ymm7,%ymm1,%ymm1");
+			asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd_low[k * 32]));
+			asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd_high[k * 32]));
+		}
+
+		/*
+		 * Reconstruct each missing block.
+		 *
+		 * Keep independent low/high accumulation chains:
+		 *
+		 * ymm0 = low accumulator
+		 * ymm1 = high accumulator
+		 * ymm2 = low table/result
+		 * ymm3 = high table/result
+		 * ymm4 = syndrome low
+		 * ymm5 = syndrome high
+		 */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/* coefficient 0 initializes the accumulators */
+			asm volatile ("vbroadcasti128 %0,%%ymm0" : : "m" (t[0][0]));
+			asm volatile ("vbroadcasti128 %0,%%ymm1" : : "m" (t[0][16]));
+
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[0]));
+			asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[0]));
+
+			asm volatile ("vpshufb %ymm4,%ymm0,%ymm0");
+			asm volatile ("vpshufb %ymm5,%ymm1,%ymm1");
+
+			for (k = 1; k < nr; ++k) {
+				asm volatile ("vbroadcasti128 %0,%%ymm2" : : "m" (t[k][0]));
+				asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (t[k][16]));
+
+				asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd_low[k * 32]));
+				asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd_high[k * 32]));
+
+				asm volatile ("vpshufb %ymm4,%ymm2,%ymm2");
+				asm volatile ("vpshufb %ymm5,%ymm3,%ymm3");
+
+				asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
+			}
+
+			asm volatile ("vpxor %ymm1,%ymm0,%ymm0");
+			asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_avx_end();
+}
+
+#ifdef CONFIG_X86_64
+/*
+ * RAID recovering for two disks AVX2 extended implementation
+ */
+static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	const int N = 2;
+	uint8_t *p[N];
+	uint8_t *pa[N];
+	uint8_t G[N * N];
+	uint8_t V[N * N];
 	size_t i;
 	int j, k;
 
+	(void)nr; /* unused, it's always 2 */
+
 	/* setup the coefficients matrix */
 	for (j = 0; j < N; ++j)
 		for (k = 0; k < N; ++k)
@@ -1227,219 +1471,845 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	raid_avx_begin();
 
-#ifdef CONFIG_X86_64
-	const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	asm volatile ("vpbroadcastb %0,%%ymm6" : : "m" (gfconst16.low4[0]));
 
-	/* precompute shuffle table pointers */
-	for (j = 0; j < N * N; ++j)
-		T[j] = &raid_gfmulpshufb[V[j]][0][0];
+	/* the inverse matrix V[] is constant for the whole recovery */
+	asm volatile ("vbroadcasti128 %0,%%ymm8" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm9" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
+	asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
+
+	for (i = 0; i < size; i += 32) {
+		/* d0 = p[0] ^ pa[0] */
+		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
+		asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
+		asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
+
+		/* d1 = p[1] ^ pa[1] */
+		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i]));
+		asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[1][i]));
+		asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
+
+		/*
+		 * Split both deltas into low/high nibbles once.
+		 *
+		 * ymm0 = d0 low
+		 * ymm2 = d0 high
+		 * ymm1 = d1 low
+		 * ymm3 = d1 high
+		 */
+		asm volatile ("vpsrlw $4,%ymm0,%ymm2");
+		asm volatile ("vpsrlw $4,%ymm1,%ymm3");
+		asm volatile ("vpand %ymm6,%ymm0,%ymm0");
+		asm volatile ("vpand %ymm6,%ymm2,%ymm2");
+		asm volatile ("vpand %ymm6,%ymm1,%ymm1");
+		asm volatile ("vpand %ymm6,%ymm3,%ymm3");
+
+		/* pa[0] = V[0] * d0 ^ V[1] * d1 */
+		asm volatile ("vpshufb %ymm0,%ymm8,%ymm4");
+		asm volatile ("vpshufb %ymm2,%ymm9,%ymm5");
+		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
+
+		asm volatile ("vpshufb %ymm1,%ymm10,%ymm5");
+		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
+		asm volatile ("vpshufb %ymm3,%ymm11,%ymm5");
+		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
+
+		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
+
+		/* pa[1] = V[2] * d0 ^ V[3] * d1 */
+		asm volatile ("vpshufb %ymm0,%ymm12,%ymm4");
+		asm volatile ("vpshufb %ymm2,%ymm13,%ymm5");
+		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
+
+		asm volatile ("vpshufb %ymm1,%ymm14,%ymm5");
+		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
+		asm volatile ("vpshufb %ymm3,%ymm15,%ymm5");
+		asm volatile ("vpxor %ymm5,%ymm4,%ymm4");
+
+		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[1][i]));
+	}
+
+	raid_avx_end();
+}
+
+/*
+ * RAID recovering AVX2 implementation optimized for up to three failures.
+ *
+ * Process 64 bytes at a time as two independent 32-byte lanes.
+ * Multiplication tables are broadcast once and shared between both lanes.
+ *
+ * Note that it uses 16 registers, meaning that x64 is required.
+ */
+static __always_inline void raid_recX_avx2ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > 3);
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, nr);
+
+	/* setup selected parity and destination pointers */
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	/* ip[] is ordered. If P is available, it is always ip[0] */
+	has_p = ip[0] == 0;
+
+	/* build the compact surviving-data list and its syndrome tables */
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	/* precompute inverse-matrix multiplication table pointers */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_avx_begin();
+
+	for (i = 0; i < size; i += 64) {
+		asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
+
+		/*
+		 * Start each syndrome with the corresponding stored parity.
+		 *
+		 * During syndrome computation:
+		 *
+		 *   ymm0/ymm1   syndrome 0, first/second 32 bytes
+		 *   ymm2/ymm3   syndrome 1, first/second 32 bytes
+		 *   ymm4/ymm5   syndrome 2, first/second 32 bytes
+		 *
+		 *   ymm6/ymm7   source low, first/second 32 bytes
+		 *   ymm8/ymm9   source high, first/second 32 bytes
+		 *   ymm12/ymm13 multiplication table / temporary
+		 *   ymm15       low-nibble mask
+		 */
+		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
+		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
+
+		if (nr >= 2) {
+			asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
+			asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i]));
+			asm volatile ("vmovdqa %0,%%ymm5" : : "m" (p[2][i + 32]));
+		}
+
+		/* add all surviving data contributions */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vmovdqa %0,%%ymm6" : : "m" (src[s][i]));
+			asm volatile ("vmovdqa %0,%%ymm7" : : "m" (src[s][i + 32]));
+
+			/*
+			 * P has coefficient 1 for every data disk.
+			 * Do the XOR before destroying the original source
+			 * while splitting it into low/high nibbles.
+			 */
+			if (has_p) {
+				asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm7,%ymm1,%ymm1");
+
+				/* split both 32-byte source lanes */
+				asm volatile ("vpsrlw $4,%ymm6,%ymm8");
+				asm volatile ("vpsrlw $4,%ymm7,%ymm9");
+				asm volatile ("vpand %ymm15,%ymm6,%ymm6");
+				asm volatile ("vpand %ymm15,%ymm7,%ymm7");
+				asm volatile ("vpand %ymm15,%ymm8,%ymm8");
+				asm volatile ("vpand %ymm15,%ymm9,%ymm9");
+			} else {
+				/* split both 32-byte source lanes */
+				asm volatile ("vpsrlw $4,%ymm6,%ymm8");
+				asm volatile ("vpsrlw $4,%ymm7,%ymm9");
+				asm volatile ("vpand %ymm15,%ymm6,%ymm6");
+				asm volatile ("vpand %ymm15,%ymm7,%ymm7");
+				asm volatile ("vpand %ymm15,%ymm8,%ymm8");
+				asm volatile ("vpand %ymm15,%ymm9,%ymm9");
+
+				/*
+				 * Syndrome 0.
+				 *
+				 * Share each table broadcast between the two
+				 * 32-byte lanes.
+				 */
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][0]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm6,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm7,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][16]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm8,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm9,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
+			}
+
+			/* syndrome 1 */
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][0]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm6,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm7,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
+				asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][16]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm8,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm9,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
+				asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
+			}
+
+			/* syndrome 2 */
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][0]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm6,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm7,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
+				asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][16]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm8,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm9,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
+				asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
+			}
+		}
+
+		/*
+		 * Split all completed syndromes.
+		 *
+		 * After this:
+		 *
+		 *   syndrome 0:
+		 *     ymm0 / ymm6   first  32 bytes low/high
+		 *     ymm1 / ymm7   second 32 bytes low/high
+		 *
+		 *   syndrome 1:
+		 *     ymm2 / ymm8   first  32 bytes low/high
+		 *     ymm3 / ymm9   second 32 bytes low/high
+		 *
+		 *   syndrome 2:
+		 *     ymm4 / ymm10  first  32 bytes low/high
+		 *     ymm5 / ymm11  second 32 bytes low/high
+		 */
+		asm volatile ("vpsrlw $4,%ymm0,%ymm6");
+		asm volatile ("vpsrlw $4,%ymm1,%ymm7");
+		asm volatile ("vpand %ymm15,%ymm0,%ymm0");
+		asm volatile ("vpand %ymm15,%ymm1,%ymm1");
+		asm volatile ("vpand %ymm15,%ymm6,%ymm6");
+		asm volatile ("vpand %ymm15,%ymm7,%ymm7");
+
+		if (nr >= 2) {
+			asm volatile ("vpsrlw $4,%ymm2,%ymm8");
+			asm volatile ("vpsrlw $4,%ymm3,%ymm9");
+			asm volatile ("vpand %ymm15,%ymm2,%ymm2");
+			asm volatile ("vpand %ymm15,%ymm3,%ymm3");
+			asm volatile ("vpand %ymm15,%ymm8,%ymm8");
+			asm volatile ("vpand %ymm15,%ymm9,%ymm9");
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vpsrlw $4,%ymm4,%ymm10");
+			asm volatile ("vpsrlw $4,%ymm5,%ymm11");
+			asm volatile ("vpand %ymm15,%ymm4,%ymm4");
+			asm volatile ("vpand %ymm15,%ymm5,%ymm5");
+			asm volatile ("vpand %ymm15,%ymm10,%ymm10");
+			asm volatile ("vpand %ymm15,%ymm11,%ymm11");
+		}
+
+		/*
+		 * Reconstruct every missing data block.
+		 *
+		 * ymm12/ymm13 are a shared multiplication table pair.
+		 * ymm14/ymm15 accumulate the two 32-byte output lanes.
+		 *
+		 * At this point ymm15 is no longer needed as the nibble mask,
+		 * so all 16 YMM registers are available.
+		 */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/*
+			 * First coefficient.
+			 * Initialize both output accumulators from the low
+			 * nibble products.
+			 */
+			asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][0]));
+			asm volatile ("vmovdqa %ymm12,%ymm13");
+			asm volatile ("vpshufb %ymm0,%ymm12,%ymm12");
+			asm volatile ("vpshufb %ymm1,%ymm13,%ymm13");
+			asm volatile ("vmovdqa %ymm12,%ymm14");
+			asm volatile ("vmovdqa %ymm13,%ymm15");
+
+			asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][16]));
+			asm volatile ("vmovdqa %ymm12,%ymm13");
+			asm volatile ("vpshufb %ymm6,%ymm12,%ymm12");
+			asm volatile ("vpshufb %ymm7,%ymm13,%ymm13");
+			asm volatile ("vpxor %ymm12,%ymm14,%ymm14");
+			asm volatile ("vpxor %ymm13,%ymm15,%ymm15");
+
+			/* second coefficient */
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][0]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm2,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm3,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm13,%ymm15,%ymm15");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[1][16]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm8,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm9,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm13,%ymm15,%ymm15");
+			}
+
+			/* third coefficient */
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][0]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm4,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm5,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm13,%ymm15,%ymm15");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[2][16]));
+				asm volatile ("vmovdqa %ymm12,%ymm13");
+				asm volatile ("vpshufb %ymm10,%ymm12,%ymm12");
+				asm volatile ("vpshufb %ymm11,%ymm13,%ymm13");
+				asm volatile ("vpxor %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm13,%ymm15,%ymm15");
+			}
+
+			asm volatile ("vmovdqa %%ymm14,%0" : "=m" (pa[j][i]));
+			asm volatile ("vmovdqa %%ymm15,%0" : "=m" (pa[j][i + 32]));
+		}
+	}
+
+	raid_avx_end();
+}
+
+/*
+ * RAID recovering AVX2 implementation.
+ *
+ * Compute only the selected syndromes, keeping them in registers.
+ * This avoids raid_delta_gen(), temporary syndrome buffers, and the
+ * generation of unused parity rows.
+ */
+static __always_inline void raid_recX_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, nr);
+
+	/* setup selected parity and destination pointers */
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	/* ip[] is ordered. If P is available, it is always ip[0] */
+	has_p = ip[0] == 0;
+
+	/* build the compact surviving-data list and its syndrome tables */
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	/* precompute inverse-matrix multiplication table pointers */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_avx_begin();
 
 	asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
 
 	for (i = 0; i < size; i += 32) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"vmovdqa (%%rax, %1), %%ymm12\n"
-			"vmovdqa (%%rbx, %1), %%ymm13\n"
-			"vpxor %%ymm13, %%ymm12, %%ymm12\n"
-			"vpsrlw $4, %%ymm12, %%ymm1\n"
-			"vpand %%ymm15, %%ymm12, %%ymm0\n"
-			"vpand %%ymm15, %%ymm1, %%ymm1\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
+		/*
+		 * Start each syndrome with the corresponding stored parity.
+		 *
+		 * During syndrome computation:
+		 *
+		 *   ymm0   syndrome 0
+		 *   ymm2   syndrome 1
+		 *   ymm4   syndrome 2
+		 *   ymm6   syndrome 3
+		 *   ymm8   syndrome 4
+		 *   ymm10  syndrome 5
+		 *
+		 *   ymm12  source / source low nibble
+		 *   ymm13  source high nibble
+		 *   ymm14  multiplication table / temporary
+		 *   ymm15  low-nibble mask
+		 */
+		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
 
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"vmovdqa (%%rax, %1), %%ymm12\n"
-			"vmovdqa (%%rbx, %1), %%ymm13\n"
-			"vpxor %%ymm13, %%ymm12, %%ymm12\n"
-			"vpsrlw $4, %%ymm12, %%ymm3\n"
-			"vpand %%ymm15, %%ymm12, %%ymm2\n"
-			"vpand %%ymm15, %%ymm3, %%ymm3\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
+		if (nr >= 2)
+			asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
 
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"vmovdqa (%%rax, %1), %%ymm12\n"
-			"vmovdqa (%%rbx, %1), %%ymm13\n"
-			"vpxor %%ymm13, %%ymm12, %%ymm12\n"
-			"vpsrlw $4, %%ymm12, %%ymm5\n"
-			"vpand %%ymm15, %%ymm12, %%ymm4\n"
-			"vpand %%ymm15, %%ymm5, %%ymm5\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
+		if (nr >= 3)
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i]));
 
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"vmovdqa (%%rax, %1), %%ymm12\n"
-			"vmovdqa (%%rbx, %1), %%ymm13\n"
-			"vpxor %%ymm13, %%ymm12, %%ymm12\n"
-			"vpsrlw $4, %%ymm12, %%ymm7\n"
-			"vpand %%ymm15, %%ymm12, %%ymm6\n"
-			"vpand %%ymm15, %%ymm7, %%ymm7\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
+		if (nr >= 4)
+			asm volatile ("vmovdqa %0,%%ymm6" : : "m" (p[3][i]));
 
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"vmovdqa (%%rax, %1), %%ymm12\n"
-			"vmovdqa (%%rbx, %1), %%ymm13\n"
-			"vpxor %%ymm13, %%ymm12, %%ymm12\n"
-			"vpsrlw $4, %%ymm12, %%ymm9\n"
-			"vpand %%ymm15, %%ymm12, %%ymm8\n"
-			"vpand %%ymm15, %%ymm9, %%ymm9\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
+		if (nr >= 5)
+			asm volatile ("vmovdqa %0,%%ymm8" : : "m" (p[4][i]));
 
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"vmovdqa (%%rax, %1), %%ymm12\n"
-			"vmovdqa (%%rbx, %1), %%ymm13\n"
-			"vpxor %%ymm13, %%ymm12, %%ymm12\n"
-			"vpsrlw $4, %%ymm12, %%ymm11\n"
-			"vpand %%ymm15, %%ymm12, %%ymm10\n"
-			"vpand %%ymm15, %%ymm11, %%ymm11\n"
+		if (nr >= 6)
+			asm volatile ("vmovdqa %0,%%ymm10" : : "m" (p[5][i]));
 
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
+		/* add all surviving data contributions */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
 
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"movq 0(%2), %%rcx\n"
-				"vbroadcasti128 0(%%rcx), %%ymm13\n"
-				"vbroadcasti128 16(%%rcx), %%ymm14\n"
-				"vpshufb %%ymm0, %%ymm13, %%ymm13\n"
-				"vpshufb %%ymm1, %%ymm14, %%ymm14\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
+			asm volatile ("vmovdqa %0,%%ymm12" : : "m" (src[s][i]));
 
-				"movq 8(%2), %%rcx\n"
-				"vbroadcasti128 0(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm2, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm13, %%ymm13\n"
-				"vbroadcasti128 16(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm3, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm14, %%ymm14\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
+			/*
+			 * P has coefficient 1 for every data disk.
+			 *
+			 * Keep the simple XOR fast path when P is the
+			 * first selected parity. Otherwise syndrome 0
+			 * uses the normal vpshufb multiplication.
+			 */
+			if (has_p) {
+				asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
 
-				"movq 16(%2), %%rcx\n"
-				"vbroadcasti128 0(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm4, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm13, %%ymm13\n"
-				"vbroadcasti128 16(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm5, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm14, %%ymm14\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
+				asm volatile ("vpsrlw $4,%ymm12,%ymm13");
+				asm volatile ("vpand %ymm15,%ymm12,%ymm12");
+				asm volatile ("vpand %ymm15,%ymm13,%ymm13");
+			} else {
+				asm volatile ("vpsrlw $4,%ymm12,%ymm13");
+				asm volatile ("vpand %ymm15,%ymm12,%ymm12");
+				asm volatile ("vpand %ymm15,%ymm13,%ymm13");
 
-				"movq 24(%2), %%rcx\n"
-				"vbroadcasti128 0(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm6, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm13, %%ymm13\n"
-				"vbroadcasti128 16(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm7, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm14, %%ymm14\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[0][0]));
+				asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm0,%ymm0");
 
-				"movq 32(%2), %%rcx\n"
-				"vbroadcasti128 0(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm8, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm13, %%ymm13\n"
-				"vbroadcasti128 16(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm9, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm14, %%ymm14\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"movq 40(%2), %%rcx\n"
-				"vbroadcasti128 0(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm10, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm13, %%ymm13\n"
-				"vbroadcasti128 16(%%rcx), %%ymm12\n"
-				"vpshufb %%ymm11, %%ymm12, %%ymm12\n"
-				"vpxor %%ymm12, %%ymm14, %%ymm14\n"
-
-				"1:\n"
-				"vpxor %%ymm14, %%ymm13, %%ymm13\n"
-				"movq %3, %%rax\n"
-				"vmovdqa %%ymm13, (%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
-				: "rax", "rcx", "cc", "memory"
-			);
-		}
-	}
-#else /* CONFIG_X86_32 */
-	uint8_t buffer_low[RAID_PARITY_MAX * 32 + 32];
-	uint8_t buffer_high[RAID_PARITY_MAX * 32 + 32];
-	uint8_t *pd_low = __align_ptr(buffer_low, 32);
-	uint8_t *pd_high = __align_ptr(buffer_high, 32);
-
-	asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
-
-	for (i = 0; i < size; i += 32) {
-		/* delta */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"vmovdqa %2, %%ymm0\n"
-				"vmovdqa %3, %%ymm1\n"
-				"vpxor   %%ymm1, %%ymm0, %%ymm0\n"
-				"vpsrlw  $4, %%ymm0, %%ymm1\n"
-				"vpand   %%ymm7, %%ymm0, %%ymm0\n"
-				"vpand   %%ymm7, %%ymm1, %%ymm1\n"
-				"vmovdqa %%ymm0, %0\n"
-				"vmovdqa %%ymm1, %1\n"
-				: "=m" (pd_low[j * 32]), "=m" (pd_high[j * 32])
-				: "m" (p[j][i]), "m" (pa[j][i])
-			);
-		}
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"vpxor %%ymm0, %%ymm0, %%ymm0\n"
-				"vpxor %%ymm1, %%ymm1, %%ymm1\n"
-			);
-
-			for (k = 0; k < N; ++k) {
-				uint8_t m = V[j * N + k];
-
-				asm volatile (
-					"vbroadcasti128 %0, %%ymm2\n"
-					"vbroadcasti128 %1, %%ymm3\n"
-					"vmovdqa        %2, %%ymm4\n"
-					"vmovdqa        %3, %%ymm5\n"
-					"vpshufb        %%ymm4, %%ymm2, %%ymm2\n"
-					"vpshufb        %%ymm5, %%ymm3, %%ymm3\n"
-					"vpxor          %%ymm2, %%ymm0, %%ymm0\n"
-					"vpxor          %%ymm3, %%ymm1, %%ymm1\n"
-					:
-					: "m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0]),
-					"m" (pd_low[k * 32]), "m" (pd_high[k * 32])
-				);
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[0][16]));
+				asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm0,%ymm0");
 			}
 
-			asm volatile (
-				"vpxor   %%ymm1, %%ymm0, %%ymm0\n"
-				"vmovdqa %%ymm0, %0\n"
-				: "=m" (pa[j][i])
-			);
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][0]));
+				asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm2,%ymm2");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][16]));
+				asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm2,%ymm2");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][0]));
+				asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm4,%ymm4");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][16]));
+				asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm4,%ymm4");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][0]));
+				asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm6,%ymm6");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][16]));
+				asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm6,%ymm6");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][0]));
+				asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm8,%ymm8");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][16]));
+				asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm8,%ymm8");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][0]));
+				asm volatile ("vpshufb %ymm12,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][16]));
+				asm volatile ("vpshufb %ymm13,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm10,%ymm10");
+			}
+		}
+
+		/*
+		 * Split every completed syndrome once.
+		 *
+		 * After this:
+		 *
+		 *   ymm0 /ymm1   delta 0 low/high
+		 *   ymm2 /ymm3   delta 1 low/high
+		 *   ymm4 /ymm5   delta 2 low/high
+		 *   ymm6 /ymm7   delta 3 low/high
+		 *   ymm8 /ymm9   delta 4 low/high
+		 *   ymm10/ymm11  delta 5 low/high
+		 */
+		asm volatile ("vpsrlw $4,%ymm0,%ymm1");
+		asm volatile ("vpand %ymm15,%ymm0,%ymm0");
+		asm volatile ("vpand %ymm15,%ymm1,%ymm1");
+
+		if (nr >= 2) {
+			asm volatile ("vpsrlw $4,%ymm2,%ymm3");
+			asm volatile ("vpand %ymm15,%ymm2,%ymm2");
+			asm volatile ("vpand %ymm15,%ymm3,%ymm3");
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vpsrlw $4,%ymm4,%ymm5");
+			asm volatile ("vpand %ymm15,%ymm4,%ymm4");
+			asm volatile ("vpand %ymm15,%ymm5,%ymm5");
+		}
+
+		if (nr >= 4) {
+			asm volatile ("vpsrlw $4,%ymm6,%ymm7");
+			asm volatile ("vpand %ymm15,%ymm6,%ymm6");
+			asm volatile ("vpand %ymm15,%ymm7,%ymm7");
+		}
+
+		if (nr >= 5) {
+			asm volatile ("vpsrlw $4,%ymm8,%ymm9");
+			asm volatile ("vpand %ymm15,%ymm8,%ymm8");
+			asm volatile ("vpand %ymm15,%ymm9,%ymm9");
+		}
+
+		if (nr >= 6) {
+			asm volatile ("vpsrlw $4,%ymm10,%ymm11");
+			asm volatile ("vpand %ymm15,%ymm10,%ymm10");
+			asm volatile ("vpand %ymm15,%ymm11,%ymm11");
+		}
+
+		/* reconstruct every missing data block */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/* first coefficient initializes the low/high accumulators */
+			asm volatile ("vbroadcasti128 %0,%%ymm12" : : "m" (t[0][0]));
+			asm volatile ("vbroadcasti128 %0,%%ymm13" : : "m" (t[0][16]));
+			asm volatile ("vpshufb %ymm0,%ymm12,%ymm12");
+			asm volatile ("vpshufb %ymm1,%ymm13,%ymm13");
+
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][0]));
+				asm volatile ("vpshufb %ymm2,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm12,%ymm12");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[1][16]));
+				asm volatile ("vpshufb %ymm3,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm13,%ymm13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][0]));
+				asm volatile ("vpshufb %ymm4,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm12,%ymm12");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[2][16]));
+				asm volatile ("vpshufb %ymm5,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm13,%ymm13");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][0]));
+				asm volatile ("vpshufb %ymm6,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm12,%ymm12");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[3][16]));
+				asm volatile ("vpshufb %ymm7,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm13,%ymm13");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][0]));
+				asm volatile ("vpshufb %ymm8,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm12,%ymm12");
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[4][16]));
+				asm volatile ("vpshufb %ymm9,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm13,%ymm13");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][0]));
+				asm volatile ("vpshufb %ymm10,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm12,%ymm12");
+
+				asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (t[5][16]));
+				asm volatile ("vpshufb %ymm11,%ymm14,%ymm14");
+				asm volatile ("vpxor %ymm14,%ymm13,%ymm13");
+			}
+
+			asm volatile ("vpxor %ymm13,%ymm12,%ymm12");
+			asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
 		}
 	}
-#endif
 
 	raid_avx_end();
 }
 
+#endif
+
+void raid_gen2_avx2_raid(int nd, size_t size, void **vv)
+{
+	raid_gen2_avx2_gen(nd, size, vv, 2);
+}
+
+void raid_gen2_avx2_aes(int nd, size_t size, void **vv)
+{
+	raid_gen2_avx2_gen(nd, size, vv, 3);
+}
+
+#ifdef CONFIG_X86_64
+void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv)
+{
+	raid_gen2_avx2ext_gen(nd, size, vv, 2);
+}
+
+void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv)
+{
+	raid_gen2_avx2ext_gen(nd, size, vv, 3);
+}
+
+void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv)
+{
+	raid_gen3_avx2ext_gen(nd, size, vv, 2);
+}
+
+void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv)
+{
+	raid_gen3_avx2ext_gen(nd, size, vv, 3);
+}
+
+void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv)
+{
+	raid_gen4_avx2ext_gen(nd, size, vv, 2);
+}
+
+void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv)
+{
+	raid_gen4_avx2ext_gen(nd, size, vv, 3);
+}
+
+/*
+ * GEN5 (penta parity with Cauchy matrix) AVX2 implementation
+ *
+ * Note that it uses 16 registers, meaning that x64 is required.
+ */
+void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv)
+{
+	raid_genX_avx2ext(nd, size, vv, 5, 2);
+}
+
+void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv)
+{
+	raid_genX_avx2ext(nd, size, vv, 5, 3);
+}
+
+/*
+ * GEN6 (hexa parity with Cauchy matrix) AVX2 implementation
+ *
+ * Note that it uses 16 registers, meaning that x64 is required.
+ */
+void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv)
+{
+	raid_genX_avx2ext(nd, size, vv, 6, 2);
+}
+
+void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv)
+{
+	raid_genX_avx2ext(nd, size, vv, 6, 3);
+}
+
+#endif
+
+void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 1);
+
+	/* if recovering with P uses the delta function */
+	if (ip[0] == 0) {
+		raid_rec1of1(id, nd, size, vv);
+		return;
+	}
+
+	raid_rec1_avx2_delta(1, id, ip, nd, size, vv);
+}
+
+void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 2);
+	raid_rec2_avx2_delta(2, id, ip, nd, size, vv);
+}
+
+void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 3);
+	raid_recX_avx2_1234(3, id, ip, nd, size, vv);
+}
+
+void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_avx2_1234(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx2(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx2(6, id, ip, nd, size, vv);
+}
+
+#ifdef CONFIG_X86_64
+void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 1);
+
+	/* if recovering with P uses the delta function */
+	if (ip[0] == 0) {
+		raid_rec1of1(id, nd, size, vv);
+		return;
+	}
+
+	/* if recovering with Q uses the delta function */
+	if (ip[0] == 1) {
+		raid_rec1_avx2_delta(1, id, ip, nd, size, vv);
+		return;
+	}
+
+	raid_recX_avx2ext_123(1, id, ip, nd, size, vv);
+}
+
+void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 2);
+
+	/* if recovering with P,Q uses the delta function */
+	if (ip[0] == 0 && ip[1] == 1) {
+		raid_rec2_avx2ext_delta(2, id, ip, nd, size, vv);
+		return;
+	}
+
+	raid_recX_avx2ext_123(2, id, ip, nd, size, vv);
+}
+
+void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 3);
+	raid_recX_avx2ext_123(3, id, ip, nd, size, vv);
+}
+
+void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_avx2ext(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx2ext(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx2ext(6, id, ip, nd, size, vv);
+}
+
+#endif
+
 void raid_register_avx2(void)
 {
 	if (raid_cpu_has_avx2()) {
@@ -1462,10 +2332,20 @@ void raid_register_avx2(void)
 
 		raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_rec1_avx2, RAID_POLY_ANY);
 		raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_rec2_avx2, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_recX_avx2, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_recX_avx2, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_recX_avx2, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_recX_avx2, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2", raid_rec3_avx2, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2", raid_rec4_avx2, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2", raid_rec5_avx2, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2", raid_rec6_avx2, RAID_POLY_ANY);
+#ifdef CONFIG_X86_64
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx2e", raid_rec1_avx2ext, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx2e", raid_rec2_avx2ext, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_rec3_avx2ext, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_rec4_avx2ext, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_rec5_avx2ext, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_rec6_avx2ext, RAID_POLY_ANY);
+
+#endif
 	}
 }
+
 #endif
diff --git a/raid/avx512.c b/raid/avx512.c
index ab7e5185..621a34bb 100644
--- a/raid/avx512.c
+++ b/raid/avx512.c
@@ -6,7 +6,6 @@
 #include "cpu.h"
 
 #ifdef CONFIG_X86_64
-
 /*
  * GEN1 (RAID5 with xor) AVX512BW implementation
  *
@@ -544,6 +543,313 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
 	raid_avx_end();
 }
 
+/*
+ * RAID recovering AVX512BW implementation.
+ *
+ * Compute only the selected syndromes directly from the stored parity
+ * and surviving data. This avoids raid_delta_gen(), temporary delta
+ * parity blocks, and generation of unused parity rows.
+ *
+ * AVX512 provides enough registers to keep all six supported syndromes
+ * in registers in a single surviving-data scan.
+ */
+static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, nr);
+
+	/* setup selected parity and destination pointers */
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	/* ip[] is ordered. If P is available, it is always ip[0] */
+	has_p = ip[0] == 0;
+
+	/*
+	 * Build the compact list of surviving data blocks and precompute
+	 * the multiplication-table pointers for each selected syndrome.
+	 */
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	/* precompute inverse-matrix multiplication table pointers */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_avx_begin();
+
+	/*
+	 * Register allocation during syndrome generation:
+	 *
+	 *   zmm0   syndrome 0
+	 *   zmm2   syndrome 1
+	 *   zmm4   syndrome 2
+	 *   zmm6   syndrome 3
+	 *   zmm8   syndrome 4
+	 *   zmm10  syndrome 5
+	 *
+	 *   zmm12  source / source low nibble
+	 *   zmm13  source high nibble
+	 *   zmm14  low multiplication table / result
+	 *   zmm15  high multiplication table / result
+	 *   zmm31  low-nibble mask
+	 *
+	 * After syndrome splitting:
+	 *
+	 *   zmm0 /zmm1   syndrome 0 low/high
+	 *   zmm2 /zmm3   syndrome 1 low/high
+	 *   zmm4 /zmm5   syndrome 2 low/high
+	 *   zmm6 /zmm7   syndrome 3 low/high
+	 *   zmm8 /zmm9   syndrome 4 low/high
+	 *   zmm10/zmm11  syndrome 5 low/high
+	 */
+
+	asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 64) {
+		/* start every selected syndrome from its stored parity block */
+		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
+
+		if (nr >= 2)
+			asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[1][i]));
+
+		if (nr >= 3)
+			asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[2][i]));
+
+		if (nr >= 4)
+			asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (p[3][i]));
+
+		if (nr >= 5)
+			asm volatile ("vmovdqa64 %0,%%zmm8" : : "m" (p[4][i]));
+
+		if (nr >= 6)
+			asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (p[5][i]));
+
+		/* add all surviving data contributions in one source scan */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vmovdqa64 %0,%%zmm12" : : "m" (src[s][i]));
+
+			/*
+			 * P has coefficient 1 for every data disk.
+			 * XOR the original source before splitting it.
+			 */
+			if (has_p) {
+				asm volatile ("vpxorq %zmm12,%zmm0,%zmm0");
+
+				asm volatile ("vpsrlw $4,%zmm12,%zmm13");
+				asm volatile ("vpandq %zmm31,%zmm12,%zmm12");
+				asm volatile ("vpandq %zmm31,%zmm13,%zmm13");
+			} else {
+				asm volatile ("vpsrlw $4,%zmm12,%zmm13");
+				asm volatile ("vpandq %zmm31,%zmm12,%zmm12");
+				asm volatile ("vpandq %zmm31,%zmm13,%zmm13");
+
+				/* syndrome 0 */
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[0][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[0][16]));
+				asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
+				asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm0");
+			}
+
+			/* syndrome 1 */
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16]));
+				asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
+				asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2");
+			}
+
+			/* syndrome 2 */
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16]));
+				asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
+				asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm4");
+			}
+
+			/* syndrome 3 */
+			if (nr >= 4) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16]));
+				asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
+				asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm6");
+			}
+
+			/* syndrome 4 */
+			if (nr >= 5) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16]));
+				asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
+				asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm8");
+			}
+
+			/* syndrome 5 */
+			if (nr >= 6) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16]));
+				asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
+				asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm10");
+			}
+		}
+
+		/* split every completed syndrome once into low/high nibbles */
+		asm volatile ("vpsrlw $4,%zmm0,%zmm1");
+		asm volatile ("vpandq %zmm31,%zmm0,%zmm0");
+		asm volatile ("vpandq %zmm31,%zmm1,%zmm1");
+
+		if (nr >= 2) {
+			asm volatile ("vpsrlw $4,%zmm2,%zmm3");
+			asm volatile ("vpandq %zmm31,%zmm2,%zmm2");
+			asm volatile ("vpandq %zmm31,%zmm3,%zmm3");
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vpsrlw $4,%zmm4,%zmm5");
+			asm volatile ("vpandq %zmm31,%zmm4,%zmm4");
+			asm volatile ("vpandq %zmm31,%zmm5,%zmm5");
+		}
+
+		if (nr >= 4) {
+			asm volatile ("vpsrlw $4,%zmm6,%zmm7");
+			asm volatile ("vpandq %zmm31,%zmm6,%zmm6");
+			asm volatile ("vpandq %zmm31,%zmm7,%zmm7");
+		}
+
+		if (nr >= 5) {
+			asm volatile ("vpsrlw $4,%zmm8,%zmm9");
+			asm volatile ("vpandq %zmm31,%zmm8,%zmm8");
+			asm volatile ("vpandq %zmm31,%zmm9,%zmm9");
+		}
+
+		if (nr >= 6) {
+			asm volatile ("vpsrlw $4,%zmm10,%zmm11");
+			asm volatile ("vpandq %zmm31,%zmm10,%zmm10");
+			asm volatile ("vpandq %zmm31,%zmm11,%zmm11");
+		}
+
+		/*
+		 * Reconstruct every missing data block.
+		 *
+		 * Keep independent low/high dependency chains:
+		 *
+		 *   zmm12 low accumulator
+		 *   zmm13 high accumulator
+		 *   zmm14 low multiplication table/result
+		 *   zmm15 high multiplication table/result
+		 *
+		 * This is intentional. Do not serialize low/high products
+		 * through a single temporary register.
+		 */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/* coefficient 0 initializes both accumulators */
+			asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (t[0][0]));
+			asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (t[0][16]));
+			asm volatile ("vpshufb %zmm0,%zmm12,%zmm12");
+			asm volatile ("vpshufb %zmm1,%zmm13,%zmm13");
+
+			if (nr >= 2) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16]));
+				asm volatile ("vpshufb %zmm2,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm3,%zmm15,%zmm15");
+				asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
+				asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16]));
+				asm volatile ("vpshufb %zmm4,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm5,%zmm15,%zmm15");
+				asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
+				asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16]));
+				asm volatile ("vpshufb %zmm6,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm7,%zmm15,%zmm15");
+				asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
+				asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16]));
+				asm volatile ("vpshufb %zmm8,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm9,%zmm15,%zmm15");
+				asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
+				asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0]));
+				asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16]));
+				asm volatile ("vpshufb %zmm10,%zmm14,%zmm14");
+				asm volatile ("vpshufb %zmm11,%zmm15,%zmm15");
+				asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
+				asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
+			}
+
+			asm volatile ("vpxorq %zmm13,%zmm12,%zmm12");
+			asm volatile ("vmovdqa64 %%zmm12,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_avx_end();
+}
+
 /*
  * GEN4 (quad parity with Cauchy matrix) AVX512BW implementation
  */
@@ -568,355 +874,47 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
 	raid_genX_avx512bw(nd, size, vv, 6);
 }
 
-/*
- * RAID recovering for one disk AVX512BW implementation
- */
 void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p, *pa;
-	uint8_t G, V;
-	size_t i;
-
-	(void)nr;
+	BUG_ON(nr != 1);
 
+	/* if recovering with P uses the delta function */
 	if (ip[0] == 0) {
 		raid_rec1of1(id, nd, size, vv);
 		return;
 	}
 
-	G = A(ip[0], id[0]);
-	V = inv(G);
-
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_avx_begin();
-
-	asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
-		asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
-		asm volatile ("vpxord   %zmm1,%zmm0,%zmm0");
-		asm volatile ("vpsrlw   $4,%zmm0,%zmm1");
-		asm volatile ("vpandd   %zmm7,%zmm0,%zmm0");
-		asm volatile ("vpandd   %zmm7,%zmm1,%zmm1");
-		asm volatile ("vpshufb  %zmm0,%zmm4,%zmm2");
-		asm volatile ("vpshufb  %zmm1,%zmm5,%zmm3");
-		asm volatile ("vpxord   %zmm3,%zmm2,%zmm2");
-		asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i]));
-	}
-
-	raid_avx_end();
+	raid_recX_avx512bw(1, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering for two disks AVX512BW implementation
- */
 void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[N];
-	uint8_t *pa[N];
-	uint8_t G[N * N];
-	uint8_t V[N * N];
-	size_t i;
-	int j, k;
-
-	(void)nr; /* unused, it's always 2 */
-
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-
-	/*
-	 * zmm7 = 0x0f nibble mask
-	 *
-	 * zmm16 = V[0] low table
-	 * zmm17 = V[0] high table
-	 * zmm18 = V[1] low table
-	 * zmm19 = V[1] high table
-	 * zmm20 = V[2] low table
-	 * zmm21 = V[2] high table
-	 * zmm22 = V[3] low table
-	 * zmm23 = V[3] high table
-	 */
-
-	asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
-
-	/* the inverse matrix V[] is constant for the whole recovery. */
-	asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
-	asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
-
-	for (i = 0; i < size; i += 64) {
-		/* d0 = p[0] ^ pa[0] */
-		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
-		asm volatile ("vpxord %zmm2,%zmm0,%zmm0");
-
-		/* d1 = p[1] ^ pa[1] */
-		asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
-		asm volatile ("vpxord %zmm3,%zmm1,%zmm1");
-
-		/*
-		 * Split both deltas into low/high nibbles once.
-		 *
-		 * zmm4  = d0 low
-		 * zmm5  = d0 high
-		 * zmm12 = d1 low
-		 * zmm13 = d1 high
-		 */
-		asm volatile ("vpsrlw $4,%zmm0,%zmm5");
-		asm volatile ("vpsrlw $4,%zmm1,%zmm13");
-		asm volatile ("vpandd %zmm7,%zmm0,%zmm4");
-		asm volatile ("vpandd %zmm7,%zmm5,%zmm5");
-		asm volatile ("vpandd %zmm7,%zmm1,%zmm12");
-		asm volatile ("vpandd %zmm7,%zmm13,%zmm13");
-
-		/*
-		 * pa[0] = V[0] * d0 ^ V[1] * d1
-		 */
-		asm volatile ("vpshufb %zmm4,%zmm16,%zmm2");
-		asm volatile ("vpshufb %zmm5,%zmm17,%zmm3");
-		asm volatile ("vpshufb %zmm12,%zmm18,%zmm10");
-		asm volatile ("vpshufb %zmm13,%zmm19,%zmm11");
-
-		asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
-		asm volatile ("vpxord %zmm10,%zmm2,%zmm2");
-		asm volatile ("vpxord %zmm11,%zmm2,%zmm2");
-
-		asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[0][i]));
-
-		/*
-		 * pa[1] = V[2] * d0 ^ V[3] * d1
-		 *
-		 * Reuse the already computed low/high nibbles.
-		 */
-		asm volatile ("vpshufb %zmm4,%zmm20,%zmm2");
-		asm volatile ("vpshufb %zmm5,%zmm21,%zmm3");
-		asm volatile ("vpshufb %zmm12,%zmm22,%zmm10");
-		asm volatile ("vpshufb %zmm13,%zmm23,%zmm11");
-
-		asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
-		asm volatile ("vpxord %zmm10,%zmm2,%zmm2");
-		asm volatile ("vpxord %zmm11,%zmm2,%zmm2");
-
-		asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[1][i]));
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 2);
+	raid_recX_avx512bw(2, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering AVX512BW implementation
- */
-void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	size_t i;
-	int j, k;
+	BUG_ON(nr != 3);
+	raid_recX_avx512bw(3, id, ip, nd, size, vv);
+}
 
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
+void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_avx512bw(4, id, ip, nd, size, vv);
+}
 
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
+void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx512bw(5, id, ip, nd, size, vv);
+}
 
-	/* precompute shuffle table pointers */
-	for (j = 0; j < N * N; ++j)
-		T[j] = &raid_gfmulpshufb[V[j]][0][0];
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-
-	asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
-
-	for (i = 0; i < size; i += 64) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm12\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm13\n"
-			"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
-			"vpsrlw $4, %%zmm12, %%zmm1\n"
-			"vpandq %%zmm31, %%zmm12, %%zmm0\n"
-			"vpandq %%zmm31, %%zmm1, %%zmm1\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
-
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm12\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm13\n"
-			"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
-			"vpsrlw $4, %%zmm12, %%zmm3\n"
-			"vpandq %%zmm31, %%zmm12, %%zmm2\n"
-			"vpandq %%zmm31, %%zmm3, %%zmm3\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
-
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm12\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm13\n"
-			"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
-			"vpsrlw $4, %%zmm12, %%zmm5\n"
-			"vpandq %%zmm31, %%zmm12, %%zmm4\n"
-			"vpandq %%zmm31, %%zmm5, %%zmm5\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
-
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm12\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm13\n"
-			"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
-			"vpsrlw $4, %%zmm12, %%zmm7\n"
-			"vpandq %%zmm31, %%zmm12, %%zmm6\n"
-			"vpandq %%zmm31, %%zmm7, %%zmm7\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
-
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm12\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm13\n"
-			"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
-			"vpsrlw $4, %%zmm12, %%zmm9\n"
-			"vpandq %%zmm31, %%zmm12, %%zmm8\n"
-			"vpandq %%zmm31, %%zmm9, %%zmm9\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
-
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm12\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm13\n"
-			"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
-			"vpsrlw $4, %%zmm12, %%zmm11\n"
-			"vpandq %%zmm31, %%zmm12, %%zmm10\n"
-			"vpandq %%zmm31, %%zmm11, %%zmm11\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"movq 0(%2), %%rcx\n"
-				"vbroadcasti32x4 0(%%rcx), %%zmm14\n"
-				"vbroadcasti32x4 16(%%rcx), %%zmm15\n"
-				"vpshufb %%zmm0, %%zmm14, %%zmm14\n"
-				"vpshufb %%zmm1, %%zmm15, %%zmm15\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
-
-				"movq 8(%2), %%rcx\n"
-				"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
-				"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
-				"vpshufb %%zmm2, %%zmm12, %%zmm12\n"
-				"vpshufb %%zmm3, %%zmm13, %%zmm13\n"
-				"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
-				"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
-
-				"movq 16(%2), %%rcx\n"
-				"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
-				"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
-				"vpshufb %%zmm4, %%zmm12, %%zmm12\n"
-				"vpshufb %%zmm5, %%zmm13, %%zmm13\n"
-				"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
-				"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
-
-				"movq 24(%2), %%rcx\n"
-				"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
-				"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
-				"vpshufb %%zmm6, %%zmm12, %%zmm12\n"
-				"vpshufb %%zmm7, %%zmm13, %%zmm13\n"
-				"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
-				"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
-
-				"movq 32(%2), %%rcx\n"
-				"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
-				"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
-				"vpshufb %%zmm8, %%zmm12, %%zmm12\n"
-				"vpshufb %%zmm9, %%zmm13, %%zmm13\n"
-				"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
-				"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"movq 40(%2), %%rcx\n"
-				"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
-				"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
-				"vpshufb %%zmm10, %%zmm12, %%zmm12\n"
-				"vpshufb %%zmm11, %%zmm13, %%zmm13\n"
-				"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
-				"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
-
-				"1:\n"
-				"vpxorq %%zmm15, %%zmm14, %%zmm14\n"
-				"movq %3, %%rax\n"
-				"vmovdqa64 %%zmm14, (%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
-				: "rax", "rcx", "cc", "memory"
-			);
-		}
-	}
-
-	raid_avx_end();
+void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx512bw(6, id, ip, nd, size, vv);
 }
 
 void raid_register_avx512(void)
@@ -932,11 +930,12 @@ void raid_register_avx512(void)
 
 			raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw, RAID_POLY_ANY);
 			raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw, RAID_POLY_ANY);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_rec3_avx512bw, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_rec4_avx512bw, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_rec5_avx512bw, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_rec6_avx512bw, RAID_POLY_ANY);
 		}
 	}
 }
+
 #endif
diff --git a/raid/gfni.c b/raid/gfni.c
index 18120646..0d91dca8 100644
--- a/raid/gfni.c
+++ b/raid/gfni.c
@@ -401,6 +401,696 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
 	raid_avx_end();
 }
 
+static __always_inline void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = raid_gfaffine_raid[A(ip[j], d)];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = raid_gfaffine_raid[V[j * nr + k]];
+
+	raid_avx_begin();
+
+	for (i = 0; i < size; i += 64) {
+		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
+		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
+
+		if (nr >= 2) {
+			asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
+			asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i]));
+			asm volatile ("vmovdqa %0,%%ymm5" : : "m" (p[2][i + 32]));
+		}
+
+		if (nr >= 4) {
+			asm volatile ("vmovdqa %0,%%ymm6" : : "m" (p[3][i]));
+			asm volatile ("vmovdqa %0,%%ymm7" : : "m" (p[3][i + 32]));
+		}
+
+		if (nr >= 5) {
+			asm volatile ("vmovdqa %0,%%ymm8" : : "m" (p[4][i]));
+			asm volatile ("vmovdqa %0,%%ymm9" : : "m" (p[4][i + 32]));
+		}
+
+		if (nr >= 6) {
+			asm volatile ("vmovdqa %0,%%ymm10" : : "m" (p[5][i]));
+			asm volatile ("vmovdqa %0,%%ymm11" : : "m" (p[5][i + 32]));
+		}
+
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vmovdqa %0,%%ymm12" : : "m" (src[s][i]));
+			asm volatile ("vmovdqa %0,%%ymm13" : : "m" (src[s][i + 32]));
+
+			if (has_p) {
+				asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
+			} else {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm0,%ymm0");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm1,%ymm1");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
+			}
+		}
+
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0]));
+			asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm12");
+			asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm13");
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm2,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm3,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm4,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm5,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm6,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm7,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm8,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm9,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0]));
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm10,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm11,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
+			asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
+		}
+	}
+
+	raid_avx_end();
+}
+
+static __always_inline void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = raid_gfaffine_raid[A(ip[j], d)];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = raid_gfaffine_raid[V[j * nr + k]];
+
+	raid_avx_begin();
+
+	for (i = 0; i < size; i += 64) {
+		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
+		if (nr >= 2)
+			asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
+		if (nr >= 3)
+			asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[2][i]));
+		if (nr >= 4)
+			asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (p[3][i]));
+		if (nr >= 5)
+			asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[4][i]));
+		if (nr >= 6)
+			asm volatile ("vmovdqa64 %0,%%zmm5" : : "m" (p[5][i]));
+
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (src[s][i]));
+
+			if (has_p) {
+				asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
+			} else {
+				asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[0][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm0,%zmm0");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[1][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm1,%zmm1");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[2][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm2,%zmm2");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[3][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm3,%zmm3");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[4][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm4,%zmm4");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (t[5][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm5,%zmm5");
+			}
+		}
+
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[0][0]));
+			asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm0,%zmm7");
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[1][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm1,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[2][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm2,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[3][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm3,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[4][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm4,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastq %0,%%zmm6" : : "m" (t[5][0]));
+				asm volatile ("vgf2p8affineqb $0,%zmm6,%zmm5,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			asm volatile ("vmovdqa64 %%zmm7,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_avx_end();
+}
+
+static __always_inline void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = A(ip[j], d);
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	raid_avx_begin();
+
+	for (i = 0; i < size; i += 64) {
+		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
+		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
+
+		if (nr >= 2) {
+			asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
+			asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vmovdqa %0,%%ymm4" : : "m" (p[2][i]));
+			asm volatile ("vmovdqa %0,%%ymm5" : : "m" (p[2][i + 32]));
+		}
+
+		if (nr >= 4) {
+			asm volatile ("vmovdqa %0,%%ymm6" : : "m" (p[3][i]));
+			asm volatile ("vmovdqa %0,%%ymm7" : : "m" (p[3][i + 32]));
+		}
+
+		if (nr >= 5) {
+			asm volatile ("vmovdqa %0,%%ymm8" : : "m" (p[4][i]));
+			asm volatile ("vmovdqa %0,%%ymm9" : : "m" (p[4][i + 32]));
+		}
+
+		if (nr >= 6) {
+			asm volatile ("vmovdqa %0,%%ymm10" : : "m" (p[5][i]));
+			asm volatile ("vmovdqa %0,%%ymm11" : : "m" (p[5][i + 32]));
+		}
+
+		for (s = 0; s < ns; ++s) {
+			asm volatile ("vmovdqa %0,%%ymm12" : : "m" (src[s][i]));
+			asm volatile ("vmovdqa %0,%%ymm13" : : "m" (src[s][i + 32]));
+
+			if (has_p) {
+				asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
+				asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
+			} else {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][0]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm0,%ymm0");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm1,%ymm1");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][1]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][2]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][3]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][4]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (S[s][5]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm12,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm13,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
+			}
+		}
+
+		for (j = 0; j < nr; ++j) {
+			asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr]));
+			asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm12");
+			asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm13");
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 1]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm2,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm3,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 2]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm4,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm5,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 3]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm6,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm7,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 4]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm8,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm9,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 5]));
+				asm volatile ("vgf2p8mulb %ymm14,%ymm10,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
+				asm volatile ("vgf2p8mulb %ymm14,%ymm11,%ymm15");
+				asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
+			}
+
+			asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
+			asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
+		}
+	}
+
+	raid_avx_end();
+}
+
+static __always_inline void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = A(ip[j], d);
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	raid_avx_begin();
+
+	for (i = 0; i < size; i += 64) {
+		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
+		if (nr >= 2)
+			asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
+		if (nr >= 3)
+			asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[2][i]));
+		if (nr >= 4)
+			asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (p[3][i]));
+		if (nr >= 5)
+			asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[4][i]));
+		if (nr >= 6)
+			asm volatile ("vmovdqa64 %0,%%zmm5" : : "m" (p[5][i]));
+
+		for (s = 0; s < ns; ++s) {
+			asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (src[s][i]));
+
+			if (has_p) {
+				asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
+			} else {
+				asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][0]));
+				asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm0,%zmm0");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][1]));
+				asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm1,%zmm1");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][2]));
+				asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm2,%zmm2");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][3]));
+				asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm3,%zmm3");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][4]));
+				asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm4,%zmm4");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (S[s][5]));
+				asm volatile ("vgf2p8mulb %zmm7,%zmm6,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm5,%zmm5");
+			}
+		}
+
+		for (j = 0; j < nr; ++j) {
+			asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr]));
+			asm volatile ("vgf2p8mulb %zmm6,%zmm0,%zmm7");
+
+			if (nr >= 2) {
+				asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 1]));
+				asm volatile ("vgf2p8mulb %zmm6,%zmm1,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 2]));
+				asm volatile ("vgf2p8mulb %zmm6,%zmm2,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 3]));
+				asm volatile ("vgf2p8mulb %zmm6,%zmm3,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 4]));
+				asm volatile ("vgf2p8mulb %zmm6,%zmm4,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vpbroadcastb %0,%%zmm6" : : "m" (V[j * nr + 5]));
+				asm volatile ("vgf2p8mulb %zmm6,%zmm5,%zmm8");
+				asm volatile ("vpxorq %zmm8,%zmm7,%zmm7");
+			}
+
+			asm volatile ("vmovdqa64 %%zmm7,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_avx_end();
+}
+
 /*
  * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation
  */
@@ -561,1078 +1251,176 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv)
 	raid_genX_avx512gfni_aes(nd, size, vv, 6);
 }
 
-/*
- * RAID recovering for one disk AVX2 GFNI implementation
- */
 void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p, *pa;
-	uint8_t G, V;
-	size_t i;
-
-	(void)nr;
+	BUG_ON(nr != 1);
 
+	/* if recovering with P uses the delta function */
 	if (ip[0] == 0) {
 		raid_rec1of1(id, nd, size, vv);
 		return;
 	}
 
-	G = A(ip[0], id[0]);
-	V = inv(G);
-
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_avx_begin();
-
-	asm volatile ("vpbroadcastq %0,%%ymm4" : : "m" (raid_gfaffine_raid[V][0]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
-		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
-
-		asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i]));
-		asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32]));
-
-		asm volatile ("vpxor    %ymm2,%ymm0,%ymm0");
-		asm volatile ("vpxor    %ymm3,%ymm1,%ymm1");
-
-		asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm0,%ymm0");
-		asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm1,%ymm1");
-
-		asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
-		asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
-	}
-
-	raid_avx_end();
+	raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering for one disk GFNI implementation
- */
-void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
-{
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p, *pa;
-	uint8_t G, V;
-	size_t i;
-
-	(void)nr;
-
-	if (ip[0] == 0) {
-		raid_rec1of1(id, nd, size, vv);
-		return;
-	}
-
-	G = A(ip[0], id[0]);
-	V = inv(G);
-
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_avx_begin();
-
-	asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfaffine_raid[V][0]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
-		asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
-		asm volatile ("vpxorq    %zmm1,%zmm0,%zmm0");
-		asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm0,%zmm0");
-		asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
-	}
-
-	raid_avx_end();
-}
-
-/*
- * RAID recovering for two disks AVX2 GFNI implementation
- */
 void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[N], *pa[N];
-	uint8_t G[N * N], V[N * N];
-	size_t i;
-	int j, k;
-
-	(void)nr;
-
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	raid_invert(G, V, N);
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-	asm volatile ("vpbroadcastq %0,%%ymm10" : : "m" (raid_gfaffine_raid[V[0]][0]));
-	asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[V[1]][0]));
-	asm volatile ("vpbroadcastq %0,%%ymm12" : : "m" (raid_gfaffine_raid[V[2]][0]));
-	asm volatile ("vpbroadcastq %0,%%ymm13" : : "m" (raid_gfaffine_raid[V[3]][0]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
-		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
-
-		asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
-		asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32]));
-
-		asm volatile ("vpxor    %ymm4,%ymm0,%ymm0");
-		asm volatile ("vpxor    %ymm5,%ymm1,%ymm1");
-
-		asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
-		asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
-
-		asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i]));
-		asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32]));
-
-		asm volatile ("vpxor    %ymm6,%ymm2,%ymm2");
-		asm volatile ("vpxor    %ymm7,%ymm3,%ymm3");
-
-		asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm0,%ymm4");
-		asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm1,%ymm5");
-
-		asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm4,%ymm4");
-		asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm5,%ymm5");
-
-		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
-		asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32]));
-
-		asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm0,%ymm6");
-		asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm1,%ymm7");
-
-		asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm2,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm6,%ymm6");
-		asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm3,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm7,%ymm7");
-
-		asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
-		asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32]));
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 2);
+	raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv);
+}
+
+void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 3);
+	raid_recX_avx2gfni_raid(3, id, ip, nd, size, vv);
+}
+
+void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_avx2gfni_raid(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx2gfni_raid(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx2gfni_raid(6, id, ip, nd, size, vv);
+}
+
+void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 1);
+
+	/* if recovering with P uses the delta function */
+	if (ip[0] == 0) {
+		raid_rec1of1(id, nd, size, vv);
+		return;
+	}
+
+	raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering for two disks GFNI implementation
- */
 void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[N], *pa[N];
-	uint8_t G[N * N], V[N * N];
-	size_t i;
-	int j, k;
-
-	(void)nr;
-
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	raid_invert(G, V, N);
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-	asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[V[0]][0]));
-	asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[V[1]][0]));
-	asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[V[2]][0]));
-	asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[V[3]][0]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
-		asm volatile ("vpxorq    %zmm2,%zmm0,%zmm0");
-		asm volatile ("vpxorq    %zmm3,%zmm1,%zmm1");
-
-		asm volatile ("vpxorq    %zmm6,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm0,%zmm2");
-		asm volatile ("vpxorq    %zmm2,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm1,%zmm3");
-		asm volatile ("vpxorq    %zmm3,%zmm6,%zmm6");
-
-		asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i]));
-
-		asm volatile ("vpxorq    %zmm6,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm0,%zmm2");
-		asm volatile ("vpxorq    %zmm2,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm1,%zmm3");
-		asm volatile ("vpxorq    %zmm3,%zmm6,%zmm6");
-
-		asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i]));
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 2);
+	raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering AVX2 GFNI implementation
- */
-void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t C[RAID_PARITY_MAX * RAID_PARITY_MAX][8];
-	size_t i;
-	int j, k;
-
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* precompute affine representations of the coefficients */
-	for (j = 0; j < N * N; ++j)
-		memcpy(C[j], raid_gfaffine_raid[V[j]], 8);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-
-	for (i = 0; i < size; i += 64) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm0\n"
-			"vmovdqa 32(%%rax, %1), %%ymm1\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm0, %%ymm0\n"
-			"vpxor %%ymm15, %%ymm1, %%ymm1\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
-
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm2\n"
-			"vmovdqa 32(%%rax, %1), %%ymm3\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm2, %%ymm2\n"
-			"vpxor %%ymm15, %%ymm3, %%ymm3\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
-
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm4\n"
-			"vmovdqa 32(%%rax, %1), %%ymm5\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm4, %%ymm4\n"
-			"vpxor %%ymm15, %%ymm5, %%ymm5\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
-
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm6\n"
-			"vmovdqa 32(%%rax, %1), %%ymm7\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm6, %%ymm6\n"
-			"vpxor %%ymm15, %%ymm7, %%ymm7\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
-
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm8\n"
-			"vmovdqa 32(%%rax, %1), %%ymm9\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm8, %%ymm8\n"
-			"vpxor %%ymm15, %%ymm9, %%ymm9\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
-
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm10\n"
-			"vmovdqa 32(%%rax, %1), %%ymm11\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm10, %%ymm10\n"
-			"vpxor %%ymm15, %%ymm11, %%ymm11\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"vpbroadcastq 0(%2), %%ymm14\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm0, %%ymm12\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm1, %%ymm13\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 8(%2), %%ymm14\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm2, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm3, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 16(%2), %%ymm14\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm4, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm5, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 24(%2), %%ymm14\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm6, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm7, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 32(%2), %%ymm14\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm8, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm9, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 40(%2), %%ymm14\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm10, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8affineqb $0, %%ymm14, %%ymm11, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-
-				"1:\n"
-				"movq %3, %%rax\n"
-				"vmovdqa %%ymm12, 0(%%rax, %1)\n"
-				"vmovdqa %%ymm13, 32(%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (C[j * N]), "r" (pa[j])
-				: "rax", "cc", "memory"
-			);
-		}
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 3);
+	raid_recX_avx512gfni_raid(3, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering GFNI implementation
- */
-void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t C[RAID_PARITY_MAX * RAID_PARITY_MAX][8];
-	size_t i;
-	int j, k;
-
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* precompute affine representations of the coefficients */
-	for (j = 0; j < N * N; ++j)
-		memcpy(C[j], raid_gfaffine_raid[V[j]], 8);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-
-	for (i = 0; i < size; i += 64) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm0\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm0, %%zmm0\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
-
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm1\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm1, %%zmm1\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
-
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm2\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm2, %%zmm2\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
-
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm3\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm3, %%zmm3\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
-
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm4\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm4, %%zmm4\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
-
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm5\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm5, %%zmm5\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"vpbroadcastq 0(%2), %%zmm6\n"
-				"vgf2p8affineqb $0, %%zmm6, %%zmm0, %%zmm7\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 8(%2), %%zmm6\n"
-				"vgf2p8affineqb $0, %%zmm6, %%zmm1, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 16(%2), %%zmm6\n"
-				"vgf2p8affineqb $0, %%zmm6, %%zmm2, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 24(%2), %%zmm6\n"
-				"vgf2p8affineqb $0, %%zmm6, %%zmm3, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 32(%2), %%zmm6\n"
-				"vgf2p8affineqb $0, %%zmm6, %%zmm4, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastq 40(%2), %%zmm6\n"
-				"vgf2p8affineqb $0, %%zmm6, %%zmm5, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-
-				"1:\n"
-				"movq %3, %%rax\n"
-				"vmovdqa64 %%zmm7, (%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (C[j * N]), "r" (pa[j])
-				: "rax", "cc", "memory"
-			);
-		}
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 4);
+	raid_recX_avx512gfni_raid(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx512gfni_raid(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx512gfni_raid(6, id, ip, nd, size, vv);
 }
 
-/*
- * AES recovering for one disk AVX2 GFNI implementation
- */
 void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p, *pa;
-	uint8_t G, V;
-	size_t i;
-
-	(void)nr;
+	BUG_ON(nr != 1);
 
+	/* if recovering with P uses the delta function */
 	if (ip[0] == 0) {
 		raid_rec1of1(id, nd, size, vv);
 		return;
 	}
 
-	G = A(ip[0], id[0]);
-	V = inv(G);
-
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_avx_begin();
-
-	asm volatile ("vpbroadcastb %0,%%ymm4" : : "m" (V));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
-		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
-
-		asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i]));
-		asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32]));
-
-		asm volatile ("vpxor    %ymm2,%ymm0,%ymm0");
-		asm volatile ("vpxor    %ymm3,%ymm1,%ymm1");
-
-		asm volatile ("vgf2p8mulb %ymm4,%ymm0,%ymm0");
-		asm volatile ("vgf2p8mulb %ymm4,%ymm1,%ymm1");
-
-		asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
-		asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
-	}
-
-	raid_avx_end();
+	raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv);
 }
 
-/*
- * AES recovering for one disk GFNI implementation
- */
-void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
-{
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p, *pa;
-	uint8_t G, V;
-	size_t i;
-
-	(void)nr;
-
-	if (ip[0] == 0) {
-		raid_rec1of1(id, nd, size, vv);
-		return;
-	}
-
-	G = A(ip[0], id[0]);
-	V = inv(G);
-
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_avx_begin();
-
-	asm volatile ("vpbroadcastb %0,%%zmm4" : : "m" (V));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
-		asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
-		asm volatile ("vpxorq    %zmm1,%zmm0,%zmm0");
-		asm volatile ("vgf2p8mulb %zmm4,%zmm0,%zmm0");
-		asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
-	}
-
-	raid_avx_end();
-}
-
-/*
- * AES recovering for two disks AVX2 GFNI implementation
- */
 void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[N], *pa[N];
-	uint8_t G[N * N], V[N * N];
-	size_t i;
-	int j, k;
-
-	(void)nr;
-
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	raid_invert(G, V, N);
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-	asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (V[0]));
-	asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (V[1]));
-	asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (V[2]));
-	asm volatile ("vpbroadcastb %0,%%ymm13" : : "m" (V[3]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
-		asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
-
-		asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
-		asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32]));
-
-		asm volatile ("vpxor    %ymm4,%ymm0,%ymm0");
-		asm volatile ("vpxor    %ymm5,%ymm1,%ymm1");
-
-		asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
-		asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
-
-		asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i]));
-		asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32]));
-
-		asm volatile ("vpxor    %ymm6,%ymm2,%ymm2");
-		asm volatile ("vpxor    %ymm7,%ymm3,%ymm3");
-
-		asm volatile ("vgf2p8mulb %ymm0,%ymm10,%ymm4");
-		asm volatile ("vgf2p8mulb %ymm1,%ymm10,%ymm5");
-
-		asm volatile ("vgf2p8mulb %ymm2,%ymm11,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm4,%ymm4");
-		asm volatile ("vgf2p8mulb %ymm3,%ymm11,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm5,%ymm5");
-
-		asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
-		asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32]));
-
-		asm volatile ("vgf2p8mulb %ymm0,%ymm12,%ymm6");
-		asm volatile ("vgf2p8mulb %ymm1,%ymm12,%ymm7");
-
-		asm volatile ("vgf2p8mulb %ymm2,%ymm13,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm6,%ymm6");
-		asm volatile ("vgf2p8mulb %ymm3,%ymm13,%ymm9");
-		asm volatile ("vpxor    %ymm9,%ymm7,%ymm7");
-
-		asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
-		asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32]));
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 2);
+	raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv);
+}
+
+void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 3);
+	raid_recX_avx2gfni_aes(3, id, ip, nd, size, vv);
+}
+
+void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_avx2gfni_aes(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx2gfni_aes(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx2gfni_aes(6, id, ip, nd, size, vv);
+}
+
+void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 1);
+
+	/* if recovering with P uses the delta function */
+	if (ip[0] == 0) {
+		raid_rec1of1(id, nd, size, vv);
+		return;
+	}
+
+	raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv);
 }
 
-/*
- * AES recovering for two disks GFNI implementation
- */
 void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[N], *pa[N];
-	uint8_t G[N * N], V[N * N];
-	size_t i;
-	int j, k;
-
-	(void)nr;
-
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	raid_invert(G, V, N);
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-	asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (V[0]));
-	asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (V[1]));
-	asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (V[2]));
-	asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (V[3]));
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
-		asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
-		asm volatile ("vpxorq    %zmm2,%zmm0,%zmm0");
-		asm volatile ("vpxorq    %zmm3,%zmm1,%zmm1");
-
-		asm volatile ("vpxorq    %zmm6,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8mulb %zmm0,%zmm8,%zmm2");
-		asm volatile ("vpxorq    %zmm2,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8mulb %zmm1,%zmm9,%zmm3");
-		asm volatile ("vpxorq    %zmm3,%zmm6,%zmm6");
-
-		asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i]));
-
-		asm volatile ("vpxorq    %zmm6,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8mulb %zmm0,%zmm10,%zmm2");
-		asm volatile ("vpxorq    %zmm2,%zmm6,%zmm6");
-
-		asm volatile ("vgf2p8mulb %zmm1,%zmm11,%zmm3");
-		asm volatile ("vpxorq    %zmm3,%zmm6,%zmm6");
-
-		asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i]));
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 2);
+	raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv);
 }
 
-/*
- * AES recovering AVX2 GFNI implementation
- */
-void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	size_t i;
-	int j, k;
-
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-
-	for (i = 0; i < size; i += 64) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm0\n"
-			"vmovdqa 32(%%rax, %1), %%ymm1\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm0, %%ymm0\n"
-			"vpxor %%ymm15, %%ymm1, %%ymm1\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
-
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm2\n"
-			"vmovdqa 32(%%rax, %1), %%ymm3\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm2, %%ymm2\n"
-			"vpxor %%ymm15, %%ymm3, %%ymm3\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
-
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm4\n"
-			"vmovdqa 32(%%rax, %1), %%ymm5\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm4, %%ymm4\n"
-			"vpxor %%ymm15, %%ymm5, %%ymm5\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
-
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm6\n"
-			"vmovdqa 32(%%rax, %1), %%ymm7\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm6, %%ymm6\n"
-			"vpxor %%ymm15, %%ymm7, %%ymm7\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
-
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm8\n"
-			"vmovdqa 32(%%rax, %1), %%ymm9\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm8, %%ymm8\n"
-			"vpxor %%ymm15, %%ymm9, %%ymm9\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
-
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"vmovdqa 0(%%rax, %1), %%ymm10\n"
-			"vmovdqa 32(%%rax, %1), %%ymm11\n"
-			"vmovdqa 0(%%rbx, %1), %%ymm14\n"
-			"vmovdqa 32(%%rbx, %1), %%ymm15\n"
-			"vpxor %%ymm14, %%ymm10, %%ymm10\n"
-			"vpxor %%ymm15, %%ymm11, %%ymm11\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"vpbroadcastb 0(%2), %%ymm14\n"
-				"vgf2p8mulb %%ymm0, %%ymm14, %%ymm12\n"
-				"vgf2p8mulb %%ymm1, %%ymm14, %%ymm13\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 1(%2), %%ymm14\n"
-				"vgf2p8mulb %%ymm2, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8mulb %%ymm3, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 2(%2), %%ymm14\n"
-				"vgf2p8mulb %%ymm4, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8mulb %%ymm5, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 3(%2), %%ymm14\n"
-				"vgf2p8mulb %%ymm6, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8mulb %%ymm7, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 4(%2), %%ymm14\n"
-				"vgf2p8mulb %%ymm8, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8mulb %%ymm9, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 5(%2), %%ymm14\n"
-				"vgf2p8mulb %%ymm10, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm12, %%ymm12\n"
-				"vgf2p8mulb %%ymm11, %%ymm14, %%ymm15\n"
-				"vpxor %%ymm15, %%ymm13, %%ymm13\n"
-
-				"1:\n"
-				"movq %3, %%rax\n"
-				"vmovdqa %%ymm12, 0(%%rax, %1)\n"
-				"vmovdqa %%ymm13, 32(%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j])
-				: "rax", "cc", "memory"
-			);
-		}
-	}
-
-	raid_avx_end();
+	BUG_ON(nr != 3);
+	raid_recX_avx512gfni_aes(3, id, ip, nd, size, vv);
 }
 
-/*
- * AES recovering GFNI implementation
- */
-void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	size_t i;
-	int j, k;
+	BUG_ON(nr != 4);
+	raid_recX_avx512gfni_aes(4, id, ip, nd, size, vv);
+}
 
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
+void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_avx512gfni_aes(5, id, ip, nd, size, vv);
+}
 
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_avx_begin();
-
-	for (i = 0; i < size; i += 64) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm0\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm0, %%zmm0\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
-
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm1\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm1, %%zmm1\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
-
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm2\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm2, %%zmm2\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
-
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm3\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm3, %%zmm3\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
-
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm4\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm4, %%zmm4\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
-
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"vmovdqa64 (%%rax, %1), %%zmm5\n"
-			"vmovdqa64 (%%rbx, %1), %%zmm6\n"
-			"vpxorq %%zmm6, %%zmm5, %%zmm5\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"vpbroadcastb 0(%2), %%zmm6\n"
-				"vgf2p8mulb %%zmm0, %%zmm6, %%zmm7\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 1(%2), %%zmm6\n"
-				"vgf2p8mulb %%zmm1, %%zmm6, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 2(%2), %%zmm6\n"
-				"vgf2p8mulb %%zmm2, %%zmm6, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 3(%2), %%zmm6\n"
-				"vgf2p8mulb %%zmm3, %%zmm6, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 4(%2), %%zmm6\n"
-				"vgf2p8mulb %%zmm4, %%zmm6, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"vpbroadcastb 5(%2), %%zmm6\n"
-				"vgf2p8mulb %%zmm5, %%zmm6, %%zmm6\n"
-				"vpxorq %%zmm6, %%zmm7, %%zmm7\n"
-
-				"1:\n"
-				"movq %3, %%rax\n"
-				"vmovdqa64 %%zmm7, (%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (&V[j * N]), "r" (pa[j])
-				: "rax", "cc", "memory"
-			);
-		}
-	}
-
-	raid_avx_end();
+void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_avx512gfni_aes(6, id, ip, nd, size, vv);
 }
 
 void raid_register_avx2gfni(void)
@@ -1652,17 +1440,17 @@ void raid_register_avx2gfni(void)
 
 		raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_raid, RAID_POLY_RAID);
 		raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_raid, RAID_POLY_RAID);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_rec3_avx2gfni_raid, RAID_POLY_RAID);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_rec4_avx2gfni_raid, RAID_POLY_RAID);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_rec5_avx2gfni_raid, RAID_POLY_RAID);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_rec6_avx2gfni_raid, RAID_POLY_RAID);
 
 		raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_aes, RAID_POLY_AES);
 		raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_aes, RAID_POLY_AES);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_rec3_avx2gfni_aes, RAID_POLY_AES);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_rec4_avx2gfni_aes, RAID_POLY_AES);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_rec5_avx2gfni_aes, RAID_POLY_AES);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_rec6_avx2gfni_aes, RAID_POLY_AES);
 	}
 }
 
@@ -1684,18 +1472,19 @@ void raid_register_avx512gfni(void)
 
 			raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_raid, RAID_POLY_RAID);
 			raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_raid, RAID_POLY_RAID);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_rec3_avx512gfni_raid, RAID_POLY_RAID);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_rec4_avx512gfni_raid, RAID_POLY_RAID);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_rec5_avx512gfni_raid, RAID_POLY_RAID);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_rec6_avx512gfni_raid, RAID_POLY_RAID);
 
 			raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_aes, RAID_POLY_AES);
 			raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_aes, RAID_POLY_AES);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
-			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_rec3_avx512gfni_aes, RAID_POLY_AES);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_rec4_avx512gfni_aes, RAID_POLY_AES);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_rec5_avx512gfni_aes, RAID_POLY_AES);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_rec6_avx512gfni_aes, RAID_POLY_AES);
 		}
 	}
 }
+
 #endif
diff --git a/raid/int.c b/raid/int.c
index 280f7d80..2036f589 100644
--- a/raid/int.c
+++ b/raid/int.c
@@ -533,7 +533,7 @@ void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	(void)nr; /* unused, it's always 1 */
 
-	/* if it's RAID5, use the faster function */
+	/* if recovering with P uses the delta function */
 	if (ip[0] == 0) {
 		raid_rec1of1(id, nd, size, vv);
 		return;
diff --git a/raid/internal.h b/raid/internal.h
index 7b459d15..1ac13e63 100644
--- a/raid/internal.h
+++ b/raid/internal.h
@@ -169,25 +169,58 @@ void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_gen1_neon(int nd, size_t size, void **vv);
 void raid_gen2_neon_raid(int nd, size_t size, void **vv);
 void raid_gen2_neon_aes(int nd, size_t size, void **vv);
@@ -202,7 +235,10 @@ void raid_gen6_neon_raid(int nd, size_t size, void **vv);
 void raid_gen6_neon_aes(int nd, size_t size, void **vv);
 void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_gen1_neon32(int nd, size_t size, void **vv);
 void raid_gen2_neon32_raid(int nd, size_t size, void **vv);
 void raid_gen2_neon32_aes(int nd, size_t size, void **vv);
@@ -217,7 +253,10 @@ void raid_gen6_neon32_raid(int nd, size_t size, void **vv);
 void raid_gen6_neon32_aes(int nd, size_t size, void **vv);
 void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
-void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
+void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
 
 /*
  * Functions for parity computation.
diff --git a/raid/neon.c b/raid/neon.c
index 0e78269b..71ed4ecd 100644
--- a/raid/neon.c
+++ b/raid/neon.c
@@ -6,7 +6,6 @@
 #include "cpu.h"
 
 #ifdef CONFIG_NEON
-
 /*
  * GEN1 (RAID5 with xor) NEON implementation
  */
@@ -94,7 +93,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
 
 		for (d = l - 1; d >= 0; --d) {
 			if (generator == 3) {
-				/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
+				/* v14-v15 are otherwise unused and preserve Q across the AES xtime */
 				asm volatile (
 					"mov v14.16b, v2.16b\n"
 					"mov v15.16b, v3.16b\n"
@@ -143,16 +142,6 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
 	raid_neon_end();
 }
 
-void raid_gen2_neon_raid(int nd, size_t size, void **vv)
-{
-	raid_gen2_neon_gen(nd, size, vv, 2);
-}
-
-void raid_gen2_neon_aes(int nd, size_t size, void **vv)
-{
-	raid_gen2_neon_gen(nd, size, vv, 3);
-}
-
 /*
  * GENz (triple parity with powers of 2^-1) NEON implementation
  */
@@ -577,6 +566,420 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
 	raid_neon_end();
 }
 
+static __always_inline void raid_recX_neon(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, nr);
+
+	/* setup selected parity and destination pointers */
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	/* ip[] is ordered. If P is available, it is always ip[0] */
+	has_p = ip[0] == 0;
+
+	/*
+	 * Build the compact list of surviving data blocks and precompute
+	 * the multiplication-table pointers for each selected syndrome.
+	 */
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	/* precompute inverse-matrix multiplication table pointers */
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_neon_begin();
+
+	for (i = 0; i < size; i += 32) {
+		/*
+		 * V31 is reused during reconstruction, so reload the nibble mask
+		 * at the beginning of every iteration.
+		 */
+		asm volatile ("ldr q31, %0" : : "m" (gfconst16.low4[0]));
+
+		/* start all selected syndromes from the stored parity */
+		asm volatile ("ldr q0, %0" : : "m" (p[0][i]));
+		asm volatile ("ldr q1, %0" : : "m" (p[0][i + 16]));
+
+		if (nr >= 2) {
+			asm volatile ("ldr q2, %0" : : "m" (p[1][i]));
+			asm volatile ("ldr q3, %0" : : "m" (p[1][i + 16]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("ldr q4, %0" : : "m" (p[2][i]));
+			asm volatile ("ldr q5, %0" : : "m" (p[2][i + 16]));
+		}
+
+		if (nr >= 4) {
+			asm volatile ("ldr q6, %0" : : "m" (p[3][i]));
+			asm volatile ("ldr q7, %0" : : "m" (p[3][i + 16]));
+		}
+
+		if (nr >= 5) {
+			asm volatile ("ldr q8, %0" : : "m" (p[4][i]));
+			asm volatile ("ldr q9, %0" : : "m" (p[4][i + 16]));
+		}
+
+		if (nr >= 6) {
+			asm volatile ("ldr q10, %0" : : "m" (p[5][i]));
+			asm volatile ("ldr q11, %0" : : "m" (p[5][i + 16]));
+		}
+
+		/*
+		 * Add all surviving data contributions.
+		 * All source reads for this chunk occur before any destination
+		 * write.
+		 */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("ldr q12, %0" : : "m" (src[s][i]));
+			asm volatile ("ldr q13, %0" : : "m" (src[s][i + 16]));
+
+			/*
+			 * P has coefficient 1. Use the original source before
+			 * destructively splitting it into nibbles.
+			 */
+			if (has_p) {
+				asm volatile ("eor v0.16b, v0.16b, v12.16b");
+				asm volatile ("eor v1.16b, v1.16b, v13.16b");
+
+				/* split the source lanes */
+				asm volatile ("ushr v14.16b, v12.16b, #4");
+				asm volatile ("ushr v15.16b, v13.16b, #4");
+				asm volatile ("and v12.16b, v12.16b, v31.16b");
+				asm volatile ("and v13.16b, v13.16b, v31.16b");
+				asm volatile ("and v14.16b, v14.16b, v31.16b");
+				asm volatile ("and v15.16b, v15.16b, v31.16b");
+			} else {
+				/* split the source lanes */
+				asm volatile ("ushr v14.16b, v12.16b, #4");
+				asm volatile ("ushr v15.16b, v13.16b, #4");
+				asm volatile ("and v12.16b, v12.16b, v31.16b");
+				asm volatile ("and v13.16b, v13.16b, v31.16b");
+				asm volatile ("and v14.16b, v14.16b, v31.16b");
+				asm volatile ("and v15.16b, v15.16b, v31.16b");
+
+				/* syndrome 0 */
+				asm volatile ("ldr q24, %0" : : "m" (t[0][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[0][16]));
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v0.16b, v0.16b, v26.16b");
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v1.16b, v1.16b, v26.16b");
+			}
+
+			/* syndrome 1 */
+			if (nr >= 2) {
+				asm volatile ("ldr q24, %0" : : "m" (t[1][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[1][16]));
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v2.16b, v2.16b, v26.16b");
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v3.16b, v3.16b, v26.16b");
+			}
+
+			/* syndrome 2 */
+			if (nr >= 3) {
+				asm volatile ("ldr q24, %0" : : "m" (t[2][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[2][16]));
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v4.16b, v4.16b, v26.16b");
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v5.16b, v5.16b, v26.16b");
+			}
+
+			/* syndrome 3 */
+			if (nr >= 4) {
+				asm volatile ("ldr q24, %0" : : "m" (t[3][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[3][16]));
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v6.16b, v6.16b, v26.16b");
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v7.16b, v7.16b, v26.16b");
+			}
+
+			/* syndrome 4 */
+			if (nr >= 5) {
+				asm volatile ("ldr q24, %0" : : "m" (t[4][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[4][16]));
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v8.16b, v8.16b, v26.16b");
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v9.16b, v9.16b, v26.16b");
+			}
+
+			/* syndrome 5 */
+			if (nr >= 6) {
+				asm volatile ("ldr q24, %0" : : "m" (t[5][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[5][16]));
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v10.16b, v10.16b, v26.16b");
+
+				asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
+				asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v26.16b, v26.16b, v27.16b");
+				asm volatile ("eor v11.16b, v11.16b, v26.16b");
+			}
+		}
+
+		/*
+		 * Expand raw syndromes backwards into:
+		 *
+		 *   S0 = v0..v3
+		 *   S1 = v4..v7
+		 *   S2 = v8..v11
+		 *   S3 = v12..v15
+		 *   S4 = v16..v19
+		 *   S5 = v20..v23
+		 */
+
+		if (nr >= 6) {
+			asm volatile ("ushr v21.16b, v10.16b, #4");
+			asm volatile ("and v20.16b, v10.16b, v31.16b");
+			asm volatile ("and v21.16b, v21.16b, v31.16b");
+			asm volatile ("ushr v23.16b, v11.16b, #4");
+			asm volatile ("and v22.16b, v11.16b, v31.16b");
+			asm volatile ("and v23.16b, v23.16b, v31.16b");
+		}
+
+		if (nr >= 5) {
+			asm volatile ("ushr v17.16b, v8.16b, #4");
+			asm volatile ("and v16.16b, v8.16b, v31.16b");
+			asm volatile ("and v17.16b, v17.16b, v31.16b");
+			asm volatile ("ushr v19.16b, v9.16b, #4");
+			asm volatile ("and v18.16b, v9.16b, v31.16b");
+			asm volatile ("and v19.16b, v19.16b, v31.16b");
+		}
+
+		if (nr >= 4) {
+			asm volatile ("ushr v13.16b, v6.16b, #4");
+			asm volatile ("and v12.16b, v6.16b, v31.16b");
+			asm volatile ("and v13.16b, v13.16b, v31.16b");
+			asm volatile ("ushr v15.16b, v7.16b, #4");
+			asm volatile ("and v14.16b, v7.16b, v31.16b");
+			asm volatile ("and v15.16b, v15.16b, v31.16b");
+		}
+
+		if (nr >= 3) {
+			asm volatile ("ushr v9.16b, v4.16b, #4");
+			asm volatile ("and v8.16b, v4.16b, v31.16b");
+			asm volatile ("and v9.16b, v9.16b, v31.16b");
+			asm volatile ("ushr v11.16b, v5.16b, #4");
+			asm volatile ("and v10.16b, v5.16b, v31.16b");
+			asm volatile ("and v11.16b, v11.16b, v31.16b");
+		}
+
+		if (nr >= 2) {
+			asm volatile ("ushr v5.16b, v2.16b, #4");
+			asm volatile ("and v4.16b, v2.16b, v31.16b");
+			asm volatile ("and v5.16b, v5.16b, v31.16b");
+			asm volatile ("ushr v7.16b, v3.16b, #4");
+			asm volatile ("and v6.16b, v3.16b, v31.16b");
+			asm volatile ("and v7.16b, v7.16b, v31.16b");
+		}
+
+		/*
+		 * S0 overlaps its original raw lane registers.
+		 * Process raw lane 1 before overwriting v1.
+		 */
+		asm volatile ("ushr v3.16b, v1.16b, #4");
+		asm volatile ("and v2.16b, v1.16b, v31.16b");
+		asm volatile ("and v3.16b, v3.16b, v31.16b");
+		asm volatile ("ushr v1.16b, v0.16b, #4");
+		asm volatile ("and v0.16b, v0.16b, v31.16b");
+		asm volatile ("and v1.16b, v1.16b, v31.16b");
+
+		/*
+		 * Reconstruct all missing blocks.
+		 *
+		 * v24/v25 = low/high tables
+		 * v26/v27 = lane-0 low/high accumulators
+		 * v28/v29 = lane-1 low/high accumulators
+		 * v30/v31 = low/high multiplication temporaries
+		 */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/* coefficient 0 initializes both lane accumulators */
+			asm volatile ("ldr q24, %0" : : "m" (t[0][0]));
+			asm volatile ("ldr q25, %0" : : "m" (t[0][16]));
+			asm volatile ("tbl v26.16b, {v24.16b}, v0.16b");
+			asm volatile ("tbl v27.16b, {v25.16b}, v1.16b");
+			asm volatile ("tbl v28.16b, {v24.16b}, v2.16b");
+			asm volatile ("tbl v29.16b, {v25.16b}, v3.16b");
+
+			if (nr >= 2) {
+				asm volatile ("ldr q24, %0" : : "m" (t[1][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[1][16]));
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v4.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v5.16b");
+				asm volatile ("eor v26.16b, v26.16b, v30.16b");
+				asm volatile ("eor v27.16b, v27.16b, v31.16b");
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v6.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v7.16b");
+				asm volatile ("eor v28.16b, v28.16b, v30.16b");
+				asm volatile ("eor v29.16b, v29.16b, v31.16b");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("ldr q24, %0" : : "m" (t[2][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[2][16]));
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v8.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v9.16b");
+				asm volatile ("eor v26.16b, v26.16b, v30.16b");
+				asm volatile ("eor v27.16b, v27.16b, v31.16b");
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v10.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v11.16b");
+				asm volatile ("eor v28.16b, v28.16b, v30.16b");
+				asm volatile ("eor v29.16b, v29.16b, v31.16b");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("ldr q24, %0" : : "m" (t[3][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[3][16]));
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v12.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v13.16b");
+				asm volatile ("eor v26.16b, v26.16b, v30.16b");
+				asm volatile ("eor v27.16b, v27.16b, v31.16b");
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v14.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v15.16b");
+				asm volatile ("eor v28.16b, v28.16b, v30.16b");
+				asm volatile ("eor v29.16b, v29.16b, v31.16b");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("ldr q24, %0" : : "m" (t[4][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[4][16]));
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v16.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v17.16b");
+				asm volatile ("eor v26.16b, v26.16b, v30.16b");
+				asm volatile ("eor v27.16b, v27.16b, v31.16b");
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v18.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v19.16b");
+				asm volatile ("eor v28.16b, v28.16b, v30.16b");
+				asm volatile ("eor v29.16b, v29.16b, v31.16b");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("ldr q24, %0" : : "m" (t[5][0]));
+				asm volatile ("ldr q25, %0" : : "m" (t[5][16]));
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v20.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v21.16b");
+				asm volatile ("eor v26.16b, v26.16b, v30.16b");
+				asm volatile ("eor v27.16b, v27.16b, v31.16b");
+
+				asm volatile ("tbl v30.16b, {v24.16b}, v22.16b");
+				asm volatile ("tbl v31.16b, {v25.16b}, v23.16b");
+				asm volatile ("eor v28.16b, v28.16b, v30.16b");
+				asm volatile ("eor v29.16b, v29.16b, v31.16b");
+			}
+
+			asm volatile ("eor v26.16b, v26.16b, v27.16b");
+			asm volatile ("eor v28.16b, v28.16b, v29.16b");
+			asm volatile ("str q26, %0" : "=m" (pa[j][i]));
+			asm volatile ("str q28, %0" : "=m" (pa[j][i + 16]));
+		}
+	}
+
+	raid_neon_end();
+}
+
+void raid_gen2_neon_raid(int nd, size_t size, void **vv)
+{
+	raid_gen2_neon_gen(nd, size, vv, 2);
+}
+
+void raid_gen2_neon_aes(int nd, size_t size, void **vv)
+{
+	raid_gen2_neon_gen(nd, size, vv, 3);
+}
+
 /*
  * GEN3 (triple parity with Cauchy matrix) NEON implementation
  */
@@ -629,431 +1032,47 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv)
 	raid_genX_neon(nd, size, vv, 6, 3);
 }
 
-/*
- * RAID recovering for one disk NEON implementation
- */
 void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p;
-	uint8_t *pa;
-	uint8_t G;
-	uint8_t V;
-	size_t i;
+	BUG_ON(nr != 1);
 
-	(void)nr; /* unused, it's always 1 */
-
-	/* if it's RAID5 uses the faster function */
+	/* if recovering with P uses the delta function */
 	if (ip[0] == 0) {
 		raid_rec1of1(id, nd, size, vv);
 		return;
 	}
 
-	/* setup the coefficients matrix */
-	G = A(ip[0], id[0]);
-
-	/* invert it to solve the system of linear equations */
-	V = inv(G);
-
-	/* compute delta parity */
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_neon_begin();
-
-	/* preload tables */
-	asm volatile (
-		"ldr q28, %0\n" /* low4 */
-		"ldr q24, %1\n" /* v low table */
-		"ldr q25, %2\n" /* v high table */
-		:
-		: "m" (gfconst16.low4[0]),
-		"m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0])
-	);
-
-	for (i = 0; i < size; i += 64) {
-		asm volatile (
-			"ldr q0, %4\n"
-			"ldr q1, %5\n"
-			"ldr q2, %6\n"
-			"ldr q3, %7\n"
-
-			"ldr q8, %8\n"
-			"ldr q9, %9\n"
-			"ldr q10, %10\n"
-			"ldr q11, %11\n"
-
-			"eor v0.16b, v0.16b, v8.16b\n"
-			"eor v1.16b, v1.16b, v9.16b\n"
-			"eor v2.16b, v2.16b, v10.16b\n"
-			"eor v3.16b, v3.16b, v11.16b\n"
-
-			"ushr v8.16b, v0.16b, #4\n"
-			"ushr v9.16b, v1.16b, #4\n"
-			"ushr v10.16b, v2.16b, #4\n"
-			"ushr v11.16b, v3.16b, #4\n"
-
-			"and v0.16b, v0.16b, v28.16b\n"
-			"and v1.16b, v1.16b, v28.16b\n"
-			"and v2.16b, v2.16b, v28.16b\n"
-			"and v3.16b, v3.16b, v28.16b\n"
-
-			"and v8.16b, v8.16b, v28.16b\n"
-			"and v9.16b, v9.16b, v28.16b\n"
-			"and v10.16b, v10.16b, v28.16b\n"
-			"and v11.16b, v11.16b, v28.16b\n"
-
-			"tbl v12.16b, {v24.16b}, v0.16b\n"
-			"tbl v13.16b, {v24.16b}, v1.16b\n"
-			"tbl v14.16b, {v24.16b}, v2.16b\n"
-			"tbl v15.16b, {v24.16b}, v3.16b\n"
-
-			"tbl v8.16b, {v25.16b}, v8.16b\n"
-			"tbl v9.16b, {v25.16b}, v9.16b\n"
-			"tbl v10.16b, {v25.16b}, v10.16b\n"
-			"tbl v11.16b, {v25.16b}, v11.16b\n"
-
-			"eor v0.16b, v12.16b, v8.16b\n"
-			"eor v1.16b, v13.16b, v9.16b\n"
-			"eor v2.16b, v14.16b, v10.16b\n"
-			"eor v3.16b, v15.16b, v11.16b\n"
-
-			"str q0, %0\n"
-			"str q1, %1\n"
-			"str q2, %2\n"
-			"str q3, %3\n"
-			: "=m" (pa[i]), "=m" (pa[i + 16]), "=m" (pa[i + 32]), "=m" (pa[i + 48])
-			: "m" (p[i]), "m" (p[i + 16]), "m" (p[i + 32]), "m" (p[i + 48]),
-			"m" (pa[i]), "m" (pa[i + 16]), "m" (pa[i + 32]), "m" (pa[i + 48])
-		);
-	}
-
-	raid_neon_end();
+	raid_recX_neon(1, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering for two disks NEON implementation
- */
 void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[2];
-	uint8_t *pa[2];
-	uint8_t G[2 * 2];
-	uint8_t V[2 * 2];
-	size_t i;
-	int j, k;
-
-	(void)nr; /* unused, it's always 2 */
-
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_neon_begin();
-
-	/* preload tables */
-	asm volatile (
-		"ldr q28, %0\n" /* low4 */
-		"ldr q20, %1\n" /* v[0] low table */
-		"ldr q21, %2\n" /* v[0] high table */
-		"ldr q22, %3\n" /* v[1] low table */
-		"ldr q23, %4\n" /* v[1] high table */
-		"ldr q24, %5\n" /* v[2] low table */
-		"ldr q25, %6\n" /* v[2] high table */
-		"ldr q26, %7\n" /* v[3] low table */
-		"ldr q27, %8\n" /* v[3] high table */
-		:
-		: "m" (gfconst16.low4[0]),
-		"m" (raid_gfmulpshufb[V[0]][0][0]), "m" (raid_gfmulpshufb[V[0]][1][0]),
-		"m" (raid_gfmulpshufb[V[1]][0][0]), "m" (raid_gfmulpshufb[V[1]][1][0]),
-		"m" (raid_gfmulpshufb[V[2]][0][0]), "m" (raid_gfmulpshufb[V[2]][1][0]),
-		"m" (raid_gfmulpshufb[V[3]][0][0]), "m" (raid_gfmulpshufb[V[3]][1][0])
-	);
-
-	for (i = 0; i < size; i += 32) {
-		asm volatile (
-			/* load delta 0 block 0 & 1 */
-			"ldr q0, %4\n"
-			"ldr q2, %5\n"
-			"ldr q8, %8\n"
-			"ldr q9, %9\n"
-			"eor v0.16b, v0.16b, v8.16b\n"
-			"eor v2.16b, v2.16b, v9.16b\n"
-
-			/* load delta 1 block 0 & 1 */
-			"ldr q4, %6\n"
-			"ldr q6, %7\n"
-			"ldr q10, %10\n"
-			"ldr q11, %11\n"
-			"eor v4.16b, v4.16b, v10.16b\n"
-			"eor v6.16b, v6.16b, v11.16b\n"
-
-			/* split delta 0 (v0, v2 -> v0 low, v1 high, v2 low, v3 high) */
-			"ushr v1.16b, v0.16b, #4\n"
-			"ushr v3.16b, v2.16b, #4\n"
-			"and v0.16b, v0.16b, v28.16b\n"
-			"and v1.16b, v1.16b, v28.16b\n"
-			"and v2.16b, v2.16b, v28.16b\n"
-			"and v3.16b, v3.16b, v28.16b\n"
-
-			/* split delta 1 (v4, v6 -> v4 low, v5 high, v6 low, v7 high) */
-			"ushr v5.16b, v4.16b, #4\n"
-			"ushr v7.16b, v6.16b, #4\n"
-			"and v4.16b, v4.16b, v28.16b\n"
-			"and v5.16b, v5.16b, v28.16b\n"
-			"and v6.16b, v6.16b, v28.16b\n"
-			"and v7.16b, v7.16b, v28.16b\n"
-
-			/* reconstruct pa[0] (v8 = block 0, v9 = block 1) */
-			/* V[0] * delta 0 */
-			"tbl v8.16b, {v20.16b}, v0.16b\n"
-			"tbl v9.16b, {v20.16b}, v2.16b\n"
-			"tbl v12.16b, {v21.16b}, v1.16b\n"
-			"tbl v13.16b, {v21.16b}, v3.16b\n"
-			"eor v8.16b, v8.16b, v12.16b\n"
-			"eor v9.16b, v9.16b, v13.16b\n"
-
-			/* V[1] * delta 1 */
-			"tbl v12.16b, {v22.16b}, v4.16b\n"
-			"tbl v13.16b, {v22.16b}, v6.16b\n"
-			"tbl v14.16b, {v23.16b}, v5.16b\n"
-			"tbl v15.16b, {v23.16b}, v7.16b\n"
-			"eor v12.16b, v12.16b, v14.16b\n"
-			"eor v13.16b, v13.16b, v15.16b\n"
-			"eor v8.16b, v8.16b, v12.16b\n"
-			"eor v9.16b, v9.16b, v13.16b\n"
-			"str q8, %0\n"
-			"str q9, %1\n"
-
-			/* reconstruct pa[1] (v10 = block 0, v11 = block 1) */
-			/* V[2] * delta 0 */
-			"tbl v10.16b, {v24.16b}, v0.16b\n"
-			"tbl v11.16b, {v24.16b}, v2.16b\n"
-			"tbl v12.16b, {v25.16b}, v1.16b\n"
-			"tbl v13.16b, {v25.16b}, v3.16b\n"
-			"eor v10.16b, v10.16b, v12.16b\n"
-			"eor v11.16b, v11.16b, v13.16b\n"
-
-			/* V[3] * delta 1 */
-			"tbl v12.16b, {v26.16b}, v4.16b\n"
-			"tbl v13.16b, {v26.16b}, v6.16b\n"
-			"tbl v14.16b, {v27.16b}, v5.16b\n"
-			"tbl v15.16b, {v27.16b}, v7.16b\n"
-			"eor v12.16b, v12.16b, v14.16b\n"
-			"eor v13.16b, v13.16b, v15.16b\n"
-			"eor v10.16b, v10.16b, v12.16b\n"
-			"eor v11.16b, v11.16b, v13.16b\n"
-			"str q10, %2\n"
-			"str q11, %3\n"
-			: "=m" (pa[0][i]), "=m" (pa[0][i + 16]),
-			"=m" (pa[1][i]), "=m" (pa[1][i + 16])
-			: "m" (p[0][i]), "m" (p[0][i + 16]),
-			"m" (p[1][i]), "m" (p[1][i + 16]),
-			"m" (pa[0][i]), "m" (pa[0][i + 16]),
-			"m" (pa[1][i]), "m" (pa[1][i + 16])
-		);
-	}
-
-	raid_neon_end();
+	BUG_ON(nr != 2);
+	raid_recX_neon(2, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering NEON implementation
- */
-void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	size_t i;
-	int j, k;
+	BUG_ON(nr != 3);
+	raid_recX_neon(3, id, ip, nd, size, vv);
+}
 
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
+void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_neon(4, id, ip, nd, size, vv);
+}
 
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
+void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_neon(5, id, ip, nd, size, vv);
+}
 
-	/* precompute shuffle table pointers */
-	for (j = 0; j < N * N; ++j)
-		T[j] = &raid_gfmulpshufb[V[j]][0][0];
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_neon_begin();
-
-	/* preload mask */
-	asm volatile (
-		"ldr q28, %0\n" /* low4 */
-		:
-		: "m" (gfconst16.low4[0])
-	);
-
-	for (i = 0; i < size; i += 16) {
-		/* delta */
-		asm volatile (
-			"ldr x4, [%2, #0]\n"
-			"ldr x5, [%3, #0]\n"
-			"ldr q0, [x4, %1]\n"
-			"ldr q14, [x5, %1]\n"
-			"eor v0.16b, v0.16b, v14.16b\n"
-			"ushr v1.16b, v0.16b, #4\n"
-			"and v0.16b, v0.16b, v28.16b\n"
-			"and v1.16b, v1.16b, v28.16b\n"
-			"cmp %0, #1\n"
-			"b.ls 1f\n"
-
-			"ldr x4, [%2, #8]\n"
-			"ldr x5, [%3, #8]\n"
-			"ldr q2, [x4, %1]\n"
-			"ldr q14, [x5, %1]\n"
-			"eor v2.16b, v2.16b, v14.16b\n"
-			"ushr v3.16b, v2.16b, #4\n"
-			"and v2.16b, v2.16b, v28.16b\n"
-			"and v3.16b, v3.16b, v28.16b\n"
-			"cmp %0, #2\n"
-			"b.ls 1f\n"
-
-			"ldr x4, [%2, #16]\n"
-			"ldr x5, [%3, #16]\n"
-			"ldr q4, [x4, %1]\n"
-			"ldr q14, [x5, %1]\n"
-			"eor v4.16b, v4.16b, v14.16b\n"
-			"ushr v5.16b, v4.16b, #4\n"
-			"and v4.16b, v4.16b, v28.16b\n"
-			"and v5.16b, v5.16b, v28.16b\n"
-			"cmp %0, #3\n"
-			"b.ls 1f\n"
-
-			"ldr x4, [%2, #24]\n"
-			"ldr x5, [%3, #24]\n"
-			"ldr q6, [x4, %1]\n"
-			"ldr q14, [x5, %1]\n"
-			"eor v6.16b, v6.16b, v14.16b\n"
-			"ushr v7.16b, v6.16b, #4\n"
-			"and v6.16b, v6.16b, v28.16b\n"
-			"and v7.16b, v7.16b, v28.16b\n"
-			"cmp %0, #4\n"
-			"b.ls 1f\n"
-
-			"ldr x4, [%2, #32]\n"
-			"ldr x5, [%3, #32]\n"
-			"ldr q8, [x4, %1]\n"
-			"ldr q14, [x5, %1]\n"
-			"eor v8.16b, v8.16b, v14.16b\n"
-			"ushr v9.16b, v8.16b, #4\n"
-			"and v8.16b, v8.16b, v28.16b\n"
-			"and v9.16b, v9.16b, v28.16b\n"
-			"cmp %0, #5\n"
-			"b.ls 1f\n"
-
-			"ldr x4, [%2, #40]\n"
-			"ldr x5, [%3, #40]\n"
-			"ldr q10, [x4, %1]\n"
-			"ldr q14, [x5, %1]\n"
-			"eor v10.16b, v10.16b, v14.16b\n"
-			"ushr v11.16b, v10.16b, #4\n"
-			"and v10.16b, v10.16b, v28.16b\n"
-			"and v11.16b, v11.16b, v28.16b\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "x4", "x5", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"ldr x5, [%2, #0]\n"
-				"ldp q24, q25, [x5]\n"
-				"tbl v12.16b, {v24.16b}, v0.16b\n"
-				"tbl v13.16b, {v25.16b}, v1.16b\n"
-				"cmp %0, #1\n"
-				"b.ls 1f\n"
-
-				"ldr x5, [%2, #8]\n"
-				"ldp q24, q25, [x5]\n"
-				"tbl v14.16b, {v24.16b}, v2.16b\n"
-				"tbl v15.16b, {v25.16b}, v3.16b\n"
-				"eor v12.16b, v12.16b, v14.16b\n"
-				"eor v13.16b, v13.16b, v15.16b\n"
-				"cmp %0, #2\n"
-				"b.ls 1f\n"
-
-				"ldr x5, [%2, #16]\n"
-				"ldp q24, q25, [x5]\n"
-				"tbl v14.16b, {v24.16b}, v4.16b\n"
-				"tbl v15.16b, {v25.16b}, v5.16b\n"
-				"eor v12.16b, v12.16b, v14.16b\n"
-				"eor v13.16b, v13.16b, v15.16b\n"
-				"cmp %0, #3\n"
-				"b.ls 1f\n"
-
-				"ldr x5, [%2, #24]\n"
-				"ldp q24, q25, [x5]\n"
-				"tbl v14.16b, {v24.16b}, v6.16b\n"
-				"tbl v15.16b, {v25.16b}, v7.16b\n"
-				"eor v12.16b, v12.16b, v14.16b\n"
-				"eor v13.16b, v13.16b, v15.16b\n"
-				"cmp %0, #4\n"
-				"b.ls 1f\n"
-
-				"ldr x5, [%2, #32]\n"
-				"ldp q24, q25, [x5]\n"
-				"tbl v14.16b, {v24.16b}, v8.16b\n"
-				"tbl v15.16b, {v25.16b}, v9.16b\n"
-				"eor v12.16b, v12.16b, v14.16b\n"
-				"eor v13.16b, v13.16b, v15.16b\n"
-				"cmp %0, #5\n"
-				"b.ls 1f\n"
-
-				"ldr x5, [%2, #40]\n"
-				"ldp q24, q25, [x5]\n"
-				"tbl v14.16b, {v24.16b}, v10.16b\n"
-				"tbl v15.16b, {v25.16b}, v11.16b\n"
-				"eor v12.16b, v12.16b, v14.16b\n"
-				"eor v13.16b, v13.16b, v15.16b\n"
-
-				"1:\n"
-				"eor v12.16b, v12.16b, v13.16b\n"
-				"str q12, [%3, %1]\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
-				: "x4", "x5", "cc", "memory"
-			);
-		}
-	}
-
-	raid_neon_end();
+void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_neon(6, id, ip, nd, size, vv);
 }
 
 void raid_register_neon(void)
@@ -1073,9 +1092,10 @@ void raid_register_neon(void)
 
 	raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY);
 	raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_rec3_neon, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_rec4_neon, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_rec5_neon, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_rec6_neon, RAID_POLY_ANY);
 }
+
 #endif
diff --git a/raid/neon32.c b/raid/neon32.c
index a959139f..04c839b1 100644
--- a/raid/neon32.c
+++ b/raid/neon32.c
@@ -6,7 +6,6 @@
 #include "cpu.h"
 
 #ifdef CONFIG_NEON32
-
 /*
  * GEN1 (RAID5 with xor) AArch32 NEON implementation
  */
@@ -63,7 +62,7 @@ void raid_gen1_neon32(int nd, size_t size, void **vv)
 }
 
 /*
- * GEN2 Cauchy AArch32 NEON implementation using the active generator.
+ * GEN2 Cauchy AArch32 NEON implementation using the active generator
  */
 static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
 	void **vv, int generator)
@@ -80,7 +79,7 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
 
 	raid_neon32_begin();
 
-	/* q14 contains the active reduction polynomial. */
+	/* q14 contains the active reduction polynomial */
 	asm volatile (
 		"vld1.8 {q14}, %0\n"
 		:
@@ -143,16 +142,6 @@ static __always_inline void raid_gen2_neon32_gen(int nd, size_t size,
 	raid_neon32_end();
 }
 
-void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
-{
-	raid_gen2_neon32_gen(nd, size, vv, 2);
-}
-
-void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
-{
-	raid_gen2_neon32_gen(nd, size, vv, 3);
-}
-
 /*
  * GENz (triple parity with powers of 2^-1) AArch32 NEON implementation
  */
@@ -241,7 +230,7 @@ void raid_genz_neon32_raid(int nd, size_t size, void **vv)
 }
 
 /*
- * GENX AArch32 NEON implementation.
+ * GENX AArch32 NEON implementation
  */
 static __always_inline void raid_genX_neon32(int nd, size_t size,
 	void **vv, int np, int generator)
@@ -261,6 +250,7 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
 
 	raid_neon32_begin();
 
+	/* generic case with at least two data disks */
 	asm volatile (
 		"vld1.8 {q14}, %0\n"
 		"vld1.8 {q15}, %1\n"
@@ -528,6 +518,648 @@ static __always_inline void raid_genX_neon32(int nd, size_t size,
 	raid_neon32_end();
 }
 
+static __always_inline void raid_recX_neon32_123(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > 3);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_neon32_begin();
+
+	for (i = 0; i < size; i += 32) {
+		/*
+		 * Q15 is the low-nibble mask during generation/splitting.
+		 * Reconstruction later reuses q15 as the second accumulator,
+		 * therefore reload it on every iteration.
+		 */
+		asm volatile ("vld1.8 {q15}, %0" : : "Q" (gfconst16.low4[0]));
+
+		/* selected stored parity */
+		asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i]));
+		asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[0][i + 16]));
+
+		if (nr >= 2) {
+			asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[1][i]));
+			asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[1][i + 16]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[2][i]));
+			asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[2][i + 16]));
+		}
+
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i]));
+			asm volatile ("vld1.8 {q7}, %0" : : "Q" (src[s][i + 16]));
+
+			if (has_p) {
+				asm volatile ("veor q0, q0, q6");
+				asm volatile ("veor q1, q1, q7");
+
+				asm volatile ("vshr.u8 q8, q6, #4");
+				asm volatile ("vshr.u8 q9, q7, #4");
+				asm volatile ("vand q6, q6, q15");
+				asm volatile ("vand q7, q7, q15");
+				asm volatile ("vand q8, q8, q15");
+				asm volatile ("vand q9, q9, q15");
+			} else {
+				asm volatile ("vshr.u8 q8, q6, #4");
+				asm volatile ("vshr.u8 q9, q7, #4");
+				asm volatile ("vand q6, q6, q15");
+				asm volatile ("vand q7, q7, q15");
+				asm volatile ("vand q8, q8, q15");
+				asm volatile ("vand q9, q9, q15");
+
+				/*
+				 * Q10 = low table
+				 * q11 = high table
+				 * q12/q13 = multiplication temporaries
+				 */
+				asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[0][0]));
+				asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[0][16]));
+
+				asm volatile ("vtbl.8 d24, {d20-d21}, d12");
+				asm volatile ("vtbl.8 d25, {d20-d21}, d13");
+				asm volatile ("vtbl.8 d26, {d22-d23}, d16");
+				asm volatile ("vtbl.8 d27, {d22-d23}, d17");
+				asm volatile ("veor q12, q12, q13");
+				asm volatile ("veor q0, q0, q12");
+
+				asm volatile ("vtbl.8 d24, {d20-d21}, d14");
+				asm volatile ("vtbl.8 d25, {d20-d21}, d15");
+				asm volatile ("vtbl.8 d26, {d22-d23}, d18");
+				asm volatile ("vtbl.8 d27, {d22-d23}, d19");
+				asm volatile ("veor q12, q12, q13");
+				asm volatile ("veor q1, q1, q12");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[1][0]));
+				asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[1][16]));
+
+				asm volatile ("vtbl.8 d24, {d20-d21}, d12");
+				asm volatile ("vtbl.8 d25, {d20-d21}, d13");
+				asm volatile ("vtbl.8 d26, {d22-d23}, d16");
+				asm volatile ("vtbl.8 d27, {d22-d23}, d17");
+				asm volatile ("veor q12, q12, q13");
+				asm volatile ("veor q2, q2, q12");
+
+				asm volatile ("vtbl.8 d24, {d20-d21}, d14");
+				asm volatile ("vtbl.8 d25, {d20-d21}, d15");
+				asm volatile ("vtbl.8 d26, {d22-d23}, d18");
+				asm volatile ("vtbl.8 d27, {d22-d23}, d19");
+				asm volatile ("veor q12, q12, q13");
+				asm volatile ("veor q3, q3, q12");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vld1.8 {q10}, %0" : : "Q" (t[2][0]));
+				asm volatile ("vld1.8 {q11}, %0" : : "Q" (t[2][16]));
+
+				asm volatile ("vtbl.8 d24, {d20-d21}, d12");
+				asm volatile ("vtbl.8 d25, {d20-d21}, d13");
+				asm volatile ("vtbl.8 d26, {d22-d23}, d16");
+				asm volatile ("vtbl.8 d27, {d22-d23}, d17");
+				asm volatile ("veor q12, q12, q13");
+				asm volatile ("veor q4, q4, q12");
+
+				asm volatile ("vtbl.8 d24, {d20-d21}, d14");
+				asm volatile ("vtbl.8 d25, {d20-d21}, d15");
+				asm volatile ("vtbl.8 d26, {d22-d23}, d18");
+				asm volatile ("vtbl.8 d27, {d22-d23}, d19");
+				asm volatile ("veor q12, q12, q13");
+				asm volatile ("veor q5, q5, q12");
+			}
+		}
+
+		/*
+		 * Expand raw syndromes backwards.
+		 *
+		 * Final layout:
+		 *
+		 * S0:
+		 *   q0 = lane0 low
+		 *   q1 = lane0 high
+		 *   q2 = lane1 low
+		 *   q3 = lane1 high
+		 *
+		 * S1:
+		 *   q4/q5 = lane0 low/high
+		 *   q6/q7 = lane1 low/high
+		 *
+		 * S2:
+		 *   q8/q9   = lane0 low/high
+		 *   q10/q11 = lane1 low/high
+		 */
+
+		if (nr >= 3) {
+			asm volatile ("vshr.u8 q9, q4, #4");
+			asm volatile ("vand q8, q4, q15");
+			asm volatile ("vand q9, q9, q15");
+
+			asm volatile ("vshr.u8 q11, q5, #4");
+			asm volatile ("vand q10, q5, q15");
+			asm volatile ("vand q11, q11, q15");
+		}
+
+		if (nr >= 2) {
+			asm volatile ("vshr.u8 q5, q2, #4");
+			asm volatile ("vand q4, q2, q15");
+			asm volatile ("vand q5, q5, q15");
+
+			asm volatile ("vshr.u8 q7, q3, #4");
+			asm volatile ("vand q6, q3, q15");
+			asm volatile ("vand q7, q7, q15");
+		}
+
+		/*
+		 * Raw S0 lane1 is q1, so process it before q1 is overwritten
+		 * by the high nibble of lane0.
+		 */
+		asm volatile ("vshr.u8 q3, q1, #4");
+		asm volatile ("vand q2, q1, q15");
+		asm volatile ("vand q3, q3, q15");
+
+		asm volatile ("vshr.u8 q1, q0, #4");
+		asm volatile ("vand q0, q0, q15");
+		asm volatile ("vand q1, q1, q15");
+
+		/*
+		 * Reconstruction.
+		 *
+		 * q12 = multiplication table
+		 * q13 = temporary result
+		 * q14 = lane0 accumulator
+		 * q15 = lane1 accumulator
+		 */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/* coefficient 0 initializes both accumulators */
+			asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0]));
+
+			asm volatile ("vtbl.8 d28, {d24-d25}, d0");
+			asm volatile ("vtbl.8 d29, {d24-d25}, d1");
+			asm volatile ("vtbl.8 d30, {d24-d25}, d4");
+			asm volatile ("vtbl.8 d31, {d24-d25}, d5");
+
+			asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][16]));
+
+			asm volatile ("vtbl.8 d26, {d24-d25}, d2");
+			asm volatile ("vtbl.8 d27, {d24-d25}, d3");
+			asm volatile ("veor q14, q14, q13");
+
+			asm volatile ("vtbl.8 d26, {d24-d25}, d6");
+			asm volatile ("vtbl.8 d27, {d24-d25}, d7");
+			asm volatile ("veor q15, q15, q13");
+
+			if (nr >= 2) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0]));
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d8");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d9");
+				asm volatile ("veor q14, q14, q13");
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d12");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d13");
+				asm volatile ("veor q15, q15, q13");
+
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][16]));
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d10");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d11");
+				asm volatile ("veor q14, q14, q13");
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d14");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d15");
+				asm volatile ("veor q15, q15, q13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0]));
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d16");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d17");
+				asm volatile ("veor q14, q14, q13");
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d20");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d21");
+				asm volatile ("veor q15, q15, q13");
+
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][16]));
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d18");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d19");
+				asm volatile ("veor q14, q14, q13");
+
+				asm volatile ("vtbl.8 d26, {d24-d25}, d22");
+				asm volatile ("vtbl.8 d27, {d24-d25}, d23");
+				asm volatile ("veor q15, q15, q13");
+			}
+
+			asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i]));
+			asm volatile ("vst1.8 {q15}, %0" : "=Q" (pa[j][i + 16]));
+		}
+	}
+
+	raid_neon32_end();
+}
+
+static __always_inline void raid_recX_neon32(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_neon32_begin();
+
+	for (i = 0; i < size; i += 16) {
+		/*
+		 * Q15 is the low-nibble mask during generation/splitting.
+		 * Reconstruction later reuses q15 as a multiplication temporary.
+		 */
+		asm volatile ("vld1.8 {q15}, %0" : : "Q" (gfconst16.low4[0]));
+
+		/*
+		 * Raw syndrome registers:
+		 *
+		 * q0 = syndrome 0
+		 * q1 = syndrome 1
+		 * q2 = syndrome 2
+		 * q3 = syndrome 3
+		 * q4 = syndrome 4
+		 * q5 = syndrome 5
+		 */
+
+		asm volatile ("vld1.8 {q0}, %0" : : "Q" (p[0][i]));
+
+		if (nr >= 2)
+			asm volatile ("vld1.8 {q1}, %0" : : "Q" (p[1][i]));
+
+		if (nr >= 3)
+			asm volatile ("vld1.8 {q2}, %0" : : "Q" (p[2][i]));
+
+		if (nr >= 4)
+			asm volatile ("vld1.8 {q3}, %0" : : "Q" (p[3][i]));
+
+		if (nr >= 5)
+			asm volatile ("vld1.8 {q4}, %0" : : "Q" (p[4][i]));
+
+		if (nr >= 6)
+			asm volatile ("vld1.8 {q5}, %0" : : "Q" (p[5][i]));
+
+		/*
+		 * During survivor generation:
+		 *
+		 * q6  = source low
+		 * q7  = source high
+		 * q8  = low table
+		 * q9  = high table
+		 * q10 = low result
+		 * q11 = high result
+		 */
+
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("vld1.8 {q6}, %0" : : "Q" (src[s][i]));
+
+			if (has_p) {
+				asm volatile ("veor q0, q0, q6");
+
+				asm volatile ("vshr.u8 q7, q6, #4");
+				asm volatile ("vand q6, q6, q15");
+				asm volatile ("vand q7, q7, q15");
+			} else {
+				asm volatile ("vshr.u8 q7, q6, #4");
+				asm volatile ("vand q6, q6, q15");
+				asm volatile ("vand q7, q7, q15");
+
+				asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[0][0]));
+				asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[0][16]));
+
+				asm volatile ("vtbl.8 d20, {d16-d17}, d12");
+				asm volatile ("vtbl.8 d21, {d16-d17}, d13");
+				asm volatile ("vtbl.8 d22, {d18-d19}, d14");
+				asm volatile ("vtbl.8 d23, {d18-d19}, d15");
+				asm volatile ("veor q10, q10, q11");
+				asm volatile ("veor q0, q0, q10");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[1][0]));
+				asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[1][16]));
+
+				asm volatile ("vtbl.8 d20, {d16-d17}, d12");
+				asm volatile ("vtbl.8 d21, {d16-d17}, d13");
+				asm volatile ("vtbl.8 d22, {d18-d19}, d14");
+				asm volatile ("vtbl.8 d23, {d18-d19}, d15");
+				asm volatile ("veor q10, q10, q11");
+				asm volatile ("veor q1, q1, q10");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[2][0]));
+				asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[2][16]));
+
+				asm volatile ("vtbl.8 d20, {d16-d17}, d12");
+				asm volatile ("vtbl.8 d21, {d16-d17}, d13");
+				asm volatile ("vtbl.8 d22, {d18-d19}, d14");
+				asm volatile ("vtbl.8 d23, {d18-d19}, d15");
+				asm volatile ("veor q10, q10, q11");
+				asm volatile ("veor q2, q2, q10");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[3][0]));
+				asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[3][16]));
+
+				asm volatile ("vtbl.8 d20, {d16-d17}, d12");
+				asm volatile ("vtbl.8 d21, {d16-d17}, d13");
+				asm volatile ("vtbl.8 d22, {d18-d19}, d14");
+				asm volatile ("vtbl.8 d23, {d18-d19}, d15");
+				asm volatile ("veor q10, q10, q11");
+				asm volatile ("veor q3, q3, q10");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[4][0]));
+				asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[4][16]));
+
+				asm volatile ("vtbl.8 d20, {d16-d17}, d12");
+				asm volatile ("vtbl.8 d21, {d16-d17}, d13");
+				asm volatile ("vtbl.8 d22, {d18-d19}, d14");
+				asm volatile ("vtbl.8 d23, {d18-d19}, d15");
+				asm volatile ("veor q10, q10, q11");
+				asm volatile ("veor q4, q4, q10");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vld1.8 {q8}, %0" : : "Q" (t[5][0]));
+				asm volatile ("vld1.8 {q9}, %0" : : "Q" (t[5][16]));
+
+				asm volatile ("vtbl.8 d20, {d16-d17}, d12");
+				asm volatile ("vtbl.8 d21, {d16-d17}, d13");
+				asm volatile ("vtbl.8 d22, {d18-d19}, d14");
+				asm volatile ("vtbl.8 d23, {d18-d19}, d15");
+				asm volatile ("veor q10, q10, q11");
+				asm volatile ("veor q5, q5, q10");
+			}
+		}
+
+		/*
+		 * Expand raw syndromes backwards.
+		 *
+		 * Final layout:
+		 *
+		 * q0/q1   syndrome 0 low/high
+		 * q2/q3   syndrome 1 low/high
+		 * q4/q5   syndrome 2 low/high
+		 * q6/q7   syndrome 3 low/high
+		 * q8/q9   syndrome 4 low/high
+		 * q10/q11 syndrome 5 low/high
+		 */
+
+		if (nr >= 6) {
+			asm volatile ("vshr.u8 q11, q5, #4");
+			asm volatile ("vand q10, q5, q15");
+			asm volatile ("vand q11, q11, q15");
+		}
+
+		if (nr >= 5) {
+			asm volatile ("vshr.u8 q9, q4, #4");
+			asm volatile ("vand q8, q4, q15");
+			asm volatile ("vand q9, q9, q15");
+		}
+
+		if (nr >= 4) {
+			asm volatile ("vshr.u8 q7, q3, #4");
+			asm volatile ("vand q6, q3, q15");
+			asm volatile ("vand q7, q7, q15");
+		}
+
+		if (nr >= 3) {
+			asm volatile ("vshr.u8 q5, q2, #4");
+			asm volatile ("vand q4, q2, q15");
+			asm volatile ("vand q5, q5, q15");
+		}
+
+		if (nr >= 2) {
+			asm volatile ("vshr.u8 q3, q1, #4");
+			asm volatile ("vand q2, q1, q15");
+			asm volatile ("vand q3, q3, q15");
+		}
+
+		asm volatile ("vshr.u8 q1, q0, #4");
+		asm volatile ("vand q0, q0, q15");
+		asm volatile ("vand q1, q1, q15");
+
+		/*
+		 * Reconstruction:
+		 *
+		 * q12 = low table
+		 * q13 = high table
+		 * q14 = output accumulator
+		 * q15 = multiplication temporary
+		 */
+
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			/* coefficient 0 initializes q14 */
+			asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[0][0]));
+			asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[0][16]));
+
+			asm volatile ("vtbl.8 d28, {d24-d25}, d0");
+			asm volatile ("vtbl.8 d29, {d24-d25}, d1");
+
+			asm volatile ("vtbl.8 d30, {d26-d27}, d2");
+			asm volatile ("vtbl.8 d31, {d26-d27}, d3");
+
+			asm volatile ("veor q14, q14, q15");
+
+			if (nr >= 2) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[1][0]));
+				asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[1][16]));
+
+				asm volatile ("vtbl.8 d30, {d24-d25}, d4");
+				asm volatile ("vtbl.8 d31, {d24-d25}, d5");
+				asm volatile ("veor q14, q14, q15");
+
+				asm volatile ("vtbl.8 d30, {d26-d27}, d6");
+				asm volatile ("vtbl.8 d31, {d26-d27}, d7");
+				asm volatile ("veor q14, q14, q15");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[2][0]));
+				asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[2][16]));
+
+				asm volatile ("vtbl.8 d30, {d24-d25}, d8");
+				asm volatile ("vtbl.8 d31, {d24-d25}, d9");
+				asm volatile ("veor q14, q14, q15");
+
+				asm volatile ("vtbl.8 d30, {d26-d27}, d10");
+				asm volatile ("vtbl.8 d31, {d26-d27}, d11");
+				asm volatile ("veor q14, q14, q15");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[3][0]));
+				asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[3][16]));
+
+				asm volatile ("vtbl.8 d30, {d24-d25}, d12");
+				asm volatile ("vtbl.8 d31, {d24-d25}, d13");
+				asm volatile ("veor q14, q14, q15");
+
+				asm volatile ("vtbl.8 d30, {d26-d27}, d14");
+				asm volatile ("vtbl.8 d31, {d26-d27}, d15");
+				asm volatile ("veor q14, q14, q15");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[4][0]));
+				asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[4][16]));
+
+				asm volatile ("vtbl.8 d30, {d24-d25}, d16");
+				asm volatile ("vtbl.8 d31, {d24-d25}, d17");
+				asm volatile ("veor q14, q14, q15");
+
+				asm volatile ("vtbl.8 d30, {d26-d27}, d18");
+				asm volatile ("vtbl.8 d31, {d26-d27}, d19");
+				asm volatile ("veor q14, q14, q15");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("vld1.8 {q12}, %0" : : "Q" (t[5][0]));
+				asm volatile ("vld1.8 {q13}, %0" : : "Q" (t[5][16]));
+
+				asm volatile ("vtbl.8 d30, {d24-d25}, d20");
+				asm volatile ("vtbl.8 d31, {d24-d25}, d21");
+				asm volatile ("veor q14, q14, q15");
+
+				asm volatile ("vtbl.8 d30, {d26-d27}, d22");
+				asm volatile ("vtbl.8 d31, {d26-d27}, d23");
+				asm volatile ("veor q14, q14, q15");
+			}
+
+			asm volatile ("vst1.8 {q14}, %0" : "=Q" (pa[j][i]));
+		}
+	}
+
+	raid_neon32_end();
+}
+
+void raid_gen2_neon32_raid(int nd, size_t size, void **vv)
+{
+	raid_gen2_neon32_gen(nd, size, vv, 2);
+}
+
+void raid_gen2_neon32_aes(int nd, size_t size, void **vv)
+{
+	raid_gen2_neon32_gen(nd, size, vv, 3);
+}
+
 /*
  * GEN3 (triple parity with Cauchy matrix) AArch32 NEON implementation
  */
@@ -580,333 +1212,47 @@ void raid_gen6_neon32_aes(int nd, size_t size, void **vv)
 	raid_genX_neon32(nd, size, vv, 6, 3);
 }
 
-/*
- * RAID recovering for one disk AArch32 NEON implementation
- */
-void raid_rec1_neon32(int nr, int *id, int *ip, int nd,
-	size_t size, void **vv)
+void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p;
-	uint8_t *pa;
-	uint8_t G;
-	uint8_t V;
-	size_t i;
+	BUG_ON(nr != 1);
 
-	(void)nr; /* unused, it's always 1 */
-
-	/* if it's RAID5 use the faster function */
+	/* if recovering with P uses the delta function */
 	if (ip[0] == 0) {
 		raid_rec1of1(id, nd, size, vv);
 		return;
 	}
 
-	/* setup the coefficients matrix */
-	G = A(ip[0], id[0]);
-
-	/* invert it to solve the system of linear equations */
-	V = inv(G);
-
-	/* compute delta parity */
-	raid_delta_gen(1, id, ip, nd, size, vv);
-
-	p = v[nd + ip[0]];
-	pa = v[id[0]];
-
-	raid_neon32_begin();
-
-	/*
-	 * Preload the mask and multiplication tables, just as neon.c does.
-	 *
-	 * q12 = low table
-	 * q13 = high table
-	 * q14 = low4 mask
-	 */
-	asm volatile (
-		"vld1.8 {q14}, %0\n"
-		"vld1.8 {q12}, %1\n"
-		"vld1.8 {q13}, %2\n"
-		:
-		: "Q" (gfconst16.low4[0]),
-		"Q" (raid_gfmulpshufb[V][0][0]),
-		"Q" (raid_gfmulpshufb[V][1][0])
-	);
-
-	for (i = 0; i < size; i += 32) {
-		asm volatile (
-			"vld1.8 {q0}, %2\n"
-			"vld1.8 {q1}, %3\n"
-			"vld1.8 {q2}, %4\n"
-			"vld1.8 {q3}, %5\n"
-
-			"veor q0, q0, q2\n"
-			"veor q1, q1, q3\n"
-
-			"vshr.u8 q4, q0, #4\n"
-			"vshr.u8 q5, q1, #4\n"
-
-			"vand q0, q0, q14\n"
-			"vand q1, q1, q14\n"
-			"vand q4, q4, q14\n"
-			"vand q5, q5, q14\n"
-
-			/* low nibbles */
-			"vtbl.8 d12, {d24-d25}, d0\n"
-			"vtbl.8 d13, {d24-d25}, d1\n"
-			"vtbl.8 d14, {d24-d25}, d2\n"
-			"vtbl.8 d15, {d24-d25}, d3\n"
-
-			/* high nibbles */
-			"vtbl.8 d16, {d26-d27}, d8\n"
-			"vtbl.8 d17, {d26-d27}, d9\n"
-			"vtbl.8 d18, {d26-d27}, d10\n"
-			"vtbl.8 d19, {d26-d27}, d11\n"
-
-			"veor q6, q6, q8\n"
-			"veor q7, q7, q9\n"
-
-			"vst1.8 {q6}, %0\n"
-			"vst1.8 {q7}, %1\n"
-			: "=Q" (pa[i]), "=Q" (pa[i + 16])
-			: "Q" (p[i]), "Q" (p[i + 16]),
-			"Q" (pa[i]), "Q" (pa[i + 16])
-		);
-	}
-
-	raid_neon32_end();
+	raid_recX_neon32_123(1, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering for two disks AArch32 NEON implementation
- */
-void raid_rec2_neon32(int nr, int *id, int *ip, int nd,
-	size_t size, void **vv)
+void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	const int N = 2;
-	uint8_t *p[2];
-	uint8_t *pa[2];
-	uint8_t G[2 * 2];
-	uint8_t V[2 * 2];
-	size_t i;
-	int j, k;
-
-	(void)nr; /* unused, it's always 2 */
-
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
-
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_neon32_begin();
-
-	/*
-	 * q4-q11 contain the four low/high multiplication table pairs.
-	 * q15 contains low4.
-	 */
-	asm volatile (
-		"vld1.8 {q15}, %0\n"
-
-		"vld1.8 {q4}, %1\n"
-		"vld1.8 {q5}, %2\n"
-
-		"vld1.8 {q6}, %3\n"
-		"vld1.8 {q7}, %4\n"
-
-		"vld1.8 {q8}, %5\n"
-		"vld1.8 {q9}, %6\n"
-
-		"vld1.8 {q10}, %7\n"
-		"vld1.8 {q11}, %8\n"
-		:
-		: "Q" (gfconst16.low4[0]),
-		"Q" (raid_gfmulpshufb[V[0]][0][0]),
-		"Q" (raid_gfmulpshufb[V[0]][1][0]),
-		"Q" (raid_gfmulpshufb[V[1]][0][0]),
-		"Q" (raid_gfmulpshufb[V[1]][1][0]),
-		"Q" (raid_gfmulpshufb[V[2]][0][0]),
-		"Q" (raid_gfmulpshufb[V[2]][1][0]),
-		"Q" (raid_gfmulpshufb[V[3]][0][0]),
-		"Q" (raid_gfmulpshufb[V[3]][1][0])
-	);
-
-	for (i = 0; i < size; i += 16) {
-		asm volatile (
-			/* delta 0 */
-			"vld1.8 {q0}, %2\n"
-			"vld1.8 {q12}, %4\n"
-			"veor q0, q0, q12\n"
-			"vshr.u8 q1, q0, #4\n"
-			"vand q0, q0, q15\n"
-			"vand q1, q1, q15\n"
-
-			/* delta 1 */
-			"vld1.8 {q2}, %3\n"
-			"vld1.8 {q12}, %5\n"
-			"veor q2, q2, q12\n"
-			"vshr.u8 q3, q2, #4\n"
-			"vand q2, q2, q15\n"
-			"vand q3, q3, q15\n"
-
-			/* pa[0] = V[0] * delta0 */
-
-			"vtbl.8 d24, {d8-d9}, d0\n"
-			"vtbl.8 d25, {d8-d9}, d1\n"
-			"vtbl.8 d26, {d10-d11}, d2\n"
-			"vtbl.8 d27, {d10-d11}, d3\n"
-			"veor q14, q12, q13\n"
-
-			/* ^ V[1] * delta1 */
-
-			"vtbl.8 d24, {d12-d13}, d4\n"
-			"vtbl.8 d25, {d12-d13}, d5\n"
-			"vtbl.8 d26, {d14-d15}, d6\n"
-			"vtbl.8 d27, {d14-d15}, d7\n"
-			"veor q12, q12, q13\n"
-			"veor q14, q14, q12\n"
-
-			"vst1.8 {q14}, %0\n"
-
-			/* pa[1] = V[2] * delta0 */
-
-			"vtbl.8 d24, {d16-d17}, d0\n"
-			"vtbl.8 d25, {d16-d17}, d1\n"
-			"vtbl.8 d26, {d18-d19}, d2\n"
-			"vtbl.8 d27, {d18-d19}, d3\n"
-			"veor q14, q12, q13\n"
-
-			/* ^ V[3] * delta1 */
-
-			"vtbl.8 d24, {d20-d21}, d4\n"
-			"vtbl.8 d25, {d20-d21}, d5\n"
-			"vtbl.8 d26, {d22-d23}, d6\n"
-			"vtbl.8 d27, {d22-d23}, d7\n"
-			"veor q12, q12, q13\n"
-			"veor q14, q14, q12\n"
-
-			"vst1.8 {q14}, %1\n"
-			: "=Q" (pa[0][i]), "=Q" (pa[1][i])
-			: "Q" (p[0][i]), "Q" (p[1][i]),
-			"Q" (pa[0][i]), "Q" (pa[1][i])
-		);
-	}
-
-	raid_neon32_end();
+	BUG_ON(nr != 2);
+	raid_recX_neon32_123(2, id, ip, nd, size, vv);
 }
 
-/*
- * RAID recovering AArch32 NEON implementation
- */
-void raid_recX_neon32(int nr, int *id, int *ip, int nd,
-	size_t size, void **vv)
+void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
-	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
-	uint8_t *p[RAID_PARITY_MAX];
-	uint8_t *pa[RAID_PARITY_MAX];
-	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
-	uint8_t D[RAID_PARITY_MAX][2][16] __aligned(16);
-	size_t i;
-	int j, k;
+	BUG_ON(nr != 3);
+	raid_recX_neon32_123(3, id, ip, nd, size, vv);
+}
 
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
+void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_neon32(4, id, ip, nd, size, vv);
+}
 
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
+void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_neon32(5, id, ip, nd, size, vv);
+}
 
-	/* precompute shuffle table pointers */
-	for (j = 0; j < N * N; ++j)
-		T[j] = &raid_gfmulpshufb[V[j]][0][0];
-
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
-
-	for (j = 0; j < N; ++j) {
-		p[j] = v[nd + ip[j]];
-		pa[j] = v[id[j]];
-	}
-
-	raid_neon32_begin();
-
-	/* preload mask */
-	asm volatile (
-		"vld1.8 {q15}, %0\n"
-		:
-		: "Q" (gfconst16.low4[0])
-	);
-
-	for (i = 0; i < size; i += 16) {
-		for (k = 0; k < N; ++k) {
-			asm volatile (
-				"vld1.8 {q0}, %2\n"
-				"vld1.8 {q1}, %3\n"
-				"veor q0, q0, q1\n"
-
-				"vshr.u8 q1, q0, #4\n"
-				"vand q0, q0, q15\n"
-				"vand q1, q1, q15\n"
-
-				"vst1.8 {q0}, %0\n"
-				"vst1.8 {q1}, %1\n"
-				: "=Q" (D[k][0][0]), "=Q" (D[k][1][0])
-				: "Q" (p[k][i]), "Q" (pa[k][i])
-			);
-		}
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"veor q0, q0, q0\n"
-			);
-
-			for (k = 0; k < N; ++k) {
-				asm volatile (
-					"vld1.8 {q2}, %0\n"
-					"vld1.8 {q3}, %1\n"
-
-					"vld1.8 {q4}, %2\n"
-					"vld1.8 {q5}, %3\n"
-
-					"vtbl.8 d12, {d8-d9}, d4\n"
-					"vtbl.8 d13, {d8-d9}, d5\n"
-
-					"vtbl.8 d14, {d10-d11}, d6\n"
-					"vtbl.8 d15, {d10-d11}, d7\n"
-
-					"veor q6, q6, q7\n"
-					"veor q0, q0, q6\n"
-					:
-					: "Q" (D[k][0][0]),
-					"Q" (D[k][1][0]),
-					"Q" (T[j * N + k][0]),
-					"Q" (T[j * N + k][16])
-				);
-			}
-
-			asm volatile (
-				"vst1.8 {q0}, %0\n"
-				: "=Q" (pa[j][i])
-			);
-		}
-	}
-
-	raid_neon32_end();
+void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_neon32(6, id, ip, nd, size, vv);
 }
 
 void raid_register_neon32(void)
@@ -932,10 +1278,10 @@ void raid_register_neon32(void)
 
 	raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon32", raid_rec1_neon32, RAID_POLY_ANY);
 	raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon32", raid_rec2_neon32, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_recX_neon32, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_recX_neon32, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_recX_neon32, RAID_POLY_ANY);
-	raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_recX_neon32, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon32", raid_rec3_neon32, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon32", raid_rec4_neon32, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon32", raid_rec5_neon32, RAID_POLY_ANY);
+	raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon32", raid_rec6_neon32, RAID_POLY_ANY);
 }
 
 #endif
diff --git a/raid/ssse3.c b/raid/ssse3.c
index d34dc310..922d9744 100644
--- a/raid/ssse3.c
+++ b/raid/ssse3.c
@@ -1,5 +1,5 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-// Copyright (C) 2013 Andrea Mazzoleni
+// SPDX-License-Identifier: GPL-2.0-or-later
+// Copyright (C) 2026 Andrea Mazzoleni
 
 #include "internal.h"
 #include "gf.h"
@@ -109,286 +109,6 @@ static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv,
 	raid_sse_end();
 }
 
-void raid_gen3_ssse3_raid(int nd, size_t size, void **vv)
-{
-	raid_gen3_ssse3_gen(nd, size, vv, 2);
-}
-
-void raid_gen3_ssse3_aes(int nd, size_t size, void **vv)
-{
-	raid_gen3_ssse3_gen(nd, size, vv, 3);
-}
-
-#ifdef CONFIG_X86_64
-/*
- * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
- *
- * Note that it uses 16 registers, meaning that x64 is required.
- */
-static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
-{
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p;
-	uint8_t *q;
-	uint8_t *r;
-	int d, l;
-	size_t i;
-
-	l = nd - 1;
-	p = v[nd];
-	q = v[nd + 1];
-	r = v[nd + 2];
-
-	/* special case with only one data disk */
-	if (l == 0) {
-		for (i = 0; i < 3; ++i)
-			memcpy(v[1 + i], v[0], size);
-		return;
-	}
-
-	raid_sse_begin();
-
-	/* generic case with at least two data disks */
-	asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));
-	asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));
-
-	for (i = 0; i < size; i += 32) {
-		/* last disk without the generator multiplication */
-		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
-		asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
-
-		asm volatile ("movdqa %xmm4,%xmm0");
-		asm volatile ("movdqa %xmm4,%xmm1");
-		asm volatile ("movdqa %xmm12,%xmm8");
-		asm volatile ("movdqa %xmm12,%xmm9");
-
-		asm volatile ("movdqa %xmm4,%xmm5");
-		asm volatile ("movdqa %xmm12,%xmm13");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("psrlw  $4,%xmm13");
-		asm volatile ("pand   %xmm11,%xmm4");
-		asm volatile ("pand   %xmm11,%xmm12");
-		asm volatile ("pand   %xmm11,%xmm5");
-		asm volatile ("pand   %xmm11,%xmm13");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
-		asm volatile ("movdqa %xmm2,%xmm10");
-		asm volatile ("movdqa %xmm7,%xmm15");
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm12,%xmm10");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pshufb %xmm13,%xmm15");
-		asm volatile ("pxor   %xmm7,%xmm2");
-		asm volatile ("pxor   %xmm15,%xmm10");
-
-		/* process two intermediate disks per iteration */
-		for (d = l - 1; d > 1; d -= 2) {
-			/* disk d */
-			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
-			asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
-
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
-			asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
-
-			if (generator == 3) {
-				asm volatile ("movdqa %xmm1,%xmm6");
-				asm volatile ("movdqa %xmm9,%xmm14");
-			}
-			asm volatile ("pxor %xmm5,%xmm5");
-			asm volatile ("pxor %xmm13,%xmm13");
-			asm volatile ("pcmpgtb %xmm1,%xmm5");
-			asm volatile ("pcmpgtb %xmm9,%xmm13");
-			asm volatile ("paddb %xmm1,%xmm1");
-			asm volatile ("paddb %xmm9,%xmm9");
-			asm volatile ("pand %xmm3,%xmm5");
-			asm volatile ("pand %xmm3,%xmm13");
-			asm volatile ("pxor %xmm5,%xmm1");
-			asm volatile ("pxor %xmm13,%xmm9");
-			if (generator == 3) {
-				asm volatile ("pxor %xmm6,%xmm1");
-				asm volatile ("pxor %xmm14,%xmm9");
-			}
-
-			asm volatile ("pxor %xmm4,%xmm0");
-			asm volatile ("pxor %xmm4,%xmm1");
-			asm volatile ("pxor %xmm12,%xmm8");
-			asm volatile ("pxor %xmm12,%xmm9");
-
-			asm volatile ("movdqa %xmm4,%xmm5");
-			asm volatile ("movdqa %xmm12,%xmm13");
-			asm volatile ("psrlw  $4,%xmm5");
-			asm volatile ("psrlw  $4,%xmm13");
-			asm volatile ("pand   %xmm11,%xmm4");
-			asm volatile ("pand   %xmm11,%xmm12");
-			asm volatile ("pand   %xmm11,%xmm5");
-			asm volatile ("pand   %xmm11,%xmm13");
-
-			asm volatile ("movdqa %xmm7,%xmm6");
-			asm volatile ("movdqa %xmm15,%xmm14");
-			asm volatile ("pshufb %xmm4,%xmm7");
-			asm volatile ("pshufb %xmm12,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm15");
-			asm volatile ("pshufb %xmm13,%xmm14");
-			asm volatile ("pxor   %xmm7,%xmm2");
-			asm volatile ("pxor   %xmm6,%xmm10");
-			asm volatile ("pxor   %xmm15,%xmm2");
-			asm volatile ("pxor   %xmm14,%xmm10");
-
-			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i]));
-			asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16]));
-
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
-			asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
-
-			if (generator == 3) {
-				asm volatile ("movdqa %xmm1,%xmm6");
-				asm volatile ("movdqa %xmm9,%xmm14");
-			}
-			asm volatile ("pxor %xmm5,%xmm5");
-			asm volatile ("pxor %xmm13,%xmm13");
-			asm volatile ("pcmpgtb %xmm1,%xmm5");
-			asm volatile ("pcmpgtb %xmm9,%xmm13");
-			asm volatile ("paddb %xmm1,%xmm1");
-			asm volatile ("paddb %xmm9,%xmm9");
-			asm volatile ("pand %xmm3,%xmm5");
-			asm volatile ("pand %xmm3,%xmm13");
-			asm volatile ("pxor %xmm5,%xmm1");
-			asm volatile ("pxor %xmm13,%xmm9");
-			if (generator == 3) {
-				asm volatile ("pxor %xmm6,%xmm1");
-				asm volatile ("pxor %xmm14,%xmm9");
-			}
-
-			asm volatile ("pxor %xmm4,%xmm0");
-			asm volatile ("pxor %xmm4,%xmm1");
-			asm volatile ("pxor %xmm12,%xmm8");
-			asm volatile ("pxor %xmm12,%xmm9");
-
-			asm volatile ("movdqa %xmm4,%xmm5");
-			asm volatile ("movdqa %xmm12,%xmm13");
-			asm volatile ("psrlw  $4,%xmm5");
-			asm volatile ("psrlw  $4,%xmm13");
-			asm volatile ("pand   %xmm11,%xmm4");
-			asm volatile ("pand   %xmm11,%xmm12");
-			asm volatile ("pand   %xmm11,%xmm5");
-			asm volatile ("pand   %xmm11,%xmm13");
-
-			asm volatile ("movdqa %xmm7,%xmm6");
-			asm volatile ("movdqa %xmm15,%xmm14");
-			asm volatile ("pshufb %xmm4,%xmm7");
-			asm volatile ("pshufb %xmm12,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm15");
-			asm volatile ("pshufb %xmm13,%xmm14");
-			asm volatile ("pxor   %xmm7,%xmm2");
-			asm volatile ("pxor   %xmm6,%xmm10");
-			asm volatile ("pxor   %xmm15,%xmm2");
-			asm volatile ("pxor   %xmm14,%xmm10");
-		}
-
-		if (d == 1) {
-			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[1][i]));
-			asm volatile ("movdqa %0,%%xmm12" : : "m" (v[1][i + 16]));
-
-			if (generator == 3) {
-				asm volatile ("movdqa %xmm1,%xmm6");
-				asm volatile ("movdqa %xmm9,%xmm14");
-			}
-			asm volatile ("pxor %xmm5,%xmm5");
-			asm volatile ("pxor %xmm13,%xmm13");
-			asm volatile ("pcmpgtb %xmm1,%xmm5");
-			asm volatile ("pcmpgtb %xmm9,%xmm13");
-			asm volatile ("paddb %xmm1,%xmm1");
-			asm volatile ("paddb %xmm9,%xmm9");
-			asm volatile ("pand %xmm3,%xmm5");
-			asm volatile ("pand %xmm3,%xmm13");
-			asm volatile ("pxor %xmm5,%xmm1");
-			asm volatile ("pxor %xmm13,%xmm9");
-			if (generator == 3) {
-				asm volatile ("pxor %xmm6,%xmm1");
-				asm volatile ("pxor %xmm14,%xmm9");
-			}
-
-			asm volatile ("pxor %xmm4,%xmm0");
-			asm volatile ("pxor %xmm4,%xmm1");
-			asm volatile ("pxor %xmm12,%xmm8");
-			asm volatile ("pxor %xmm12,%xmm9");
-
-			asm volatile ("movdqa %xmm4,%xmm5");
-			asm volatile ("movdqa %xmm12,%xmm13");
-			asm volatile ("psrlw  $4,%xmm5");
-			asm volatile ("psrlw  $4,%xmm13");
-			asm volatile ("pand   %xmm11,%xmm4");
-			asm volatile ("pand   %xmm11,%xmm12");
-			asm volatile ("pand   %xmm11,%xmm5");
-			asm volatile ("pand   %xmm11,%xmm13");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
-			asm volatile ("movdqa %xmm6,%xmm14");
-			asm volatile ("movdqa %xmm7,%xmm15");
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm12,%xmm14");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pshufb %xmm13,%xmm15");
-			asm volatile ("pxor   %xmm6,%xmm2");
-			asm volatile ("pxor   %xmm14,%xmm10");
-			asm volatile ("pxor   %xmm7,%xmm2");
-			asm volatile ("pxor   %xmm15,%xmm10");
-		}
-
-		/* first disk with all coefficients at 1 */
-		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
-		asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
-
-		if (generator == 3) {
-			asm volatile ("movdqa %xmm1,%xmm6");
-			asm volatile ("movdqa %xmm9,%xmm14");
-		}
-		asm volatile ("pxor %xmm5,%xmm5");
-		asm volatile ("pxor %xmm13,%xmm13");
-		asm volatile ("pcmpgtb %xmm1,%xmm5");
-		asm volatile ("pcmpgtb %xmm9,%xmm13");
-		asm volatile ("paddb %xmm1,%xmm1");
-		asm volatile ("paddb %xmm9,%xmm9");
-		asm volatile ("pand %xmm3,%xmm5");
-		asm volatile ("pand %xmm3,%xmm13");
-		asm volatile ("pxor %xmm5,%xmm1");
-		asm volatile ("pxor %xmm13,%xmm9");
-		if (generator == 3) {
-			asm volatile ("pxor %xmm6,%xmm1");
-			asm volatile ("pxor %xmm14,%xmm9");
-		}
-
-		asm volatile ("pxor %xmm4,%xmm0");
-		asm volatile ("pxor %xmm4,%xmm1");
-		asm volatile ("pxor %xmm4,%xmm2");
-		asm volatile ("pxor %xmm12,%xmm8");
-		asm volatile ("pxor %xmm12,%xmm9");
-		asm volatile ("pxor %xmm12,%xmm10");
-
-		asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
-		asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16]));
-		asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
-		asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16]));
-		asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
-		asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));
-	}
-
-	raid_sse_end();
-}
-
-void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv)
-{
-	raid_gen3_ssse3ext_gen(nd, size, vv, 2);
-}
-
-void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv)
-{
-	raid_gen3_ssse3ext_gen(nd, size, vv, 3);
-}
-#endif
-
 /*
  * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
  */
@@ -511,17 +231,556 @@ static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv,
 	raid_sse_end();
 }
 
-void raid_gen4_ssse3_raid(int nd, size_t size, void **vv)
+/*
+ * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
+ */
+void raid_gen5_ssse3_raid(int nd, size_t size, void **vv)
 {
-	raid_gen4_ssse3_gen(nd, size, vv, 2);
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p;
+	uint8_t *q;
+	uint8_t *r;
+	uint8_t *s;
+	uint8_t *t;
+	int d, l;
+	size_t i;
+	uint8_t buffer[16 + 16];
+	uint8_t *pd = __align_ptr(buffer, 16);
+
+	l = nd - 1;
+	p = v[nd];
+	q = v[nd + 1];
+	r = v[nd + 2];
+	s = v[nd + 3];
+	t = v[nd + 4];
+
+	/* special case with only one data disk */
+	if (l == 0) {
+		for (i = 0; i < 5; ++i)
+			memcpy(v[1 + i], v[0], size);
+		return;
+	}
+
+	raid_sse_begin();
+
+	/* generic case with at least two data disks */
+	for (i = 0; i < size; i += 16) {
+		/* last disk without the generator multiplication */
+		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
+
+		asm volatile ("movdqa %xmm4,%xmm0");
+		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
+
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+		asm volatile ("movdqa %xmm4,%xmm5");
+		asm volatile ("psrlw  $4,%xmm5");
+		asm volatile ("pand   %xmm7,%xmm4");
+		asm volatile ("pand   %xmm7,%xmm5");
+
+		asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm1");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm1");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm2");
+
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm3");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm3");
+
+		/* intermediate disks */
+		for (d = l - 1; d > 0; --d) {
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
+
+			asm volatile ("pxor %xmm5,%xmm5");
+			asm volatile ("pcmpgtb %xmm0,%xmm5");
+			asm volatile ("paddb %xmm0,%xmm0");
+			asm volatile ("pand %xmm7,%xmm5");
+			asm volatile ("pxor %xmm5,%xmm0");
+
+			asm volatile ("pxor %xmm4,%xmm0");
+			asm volatile ("pxor %xmm4,%xmm6");
+			asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0]));
+
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+			asm volatile ("movdqa %xmm4,%xmm5");
+			asm volatile ("psrlw  $4,%xmm5");
+			asm volatile ("pand   %xmm7,%xmm4");
+			asm volatile ("pand   %xmm7,%xmm5");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm1");
+			asm volatile ("pxor   %xmm7,%xmm1");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm2");
+			asm volatile ("pxor   %xmm7,%xmm2");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm3");
+			asm volatile ("pxor   %xmm7,%xmm3");
+		}
+
+		/* first disk with all coefficients at 1 */
+		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
+		asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
+
+		asm volatile ("pxor %xmm5,%xmm5");
+		asm volatile ("pcmpgtb %xmm0,%xmm5");
+		asm volatile ("paddb %xmm0,%xmm0");
+		asm volatile ("pand %xmm7,%xmm5");
+		asm volatile ("pxor %xmm5,%xmm0");
+
+		asm volatile ("pxor %xmm4,%xmm0");
+		asm volatile ("pxor %xmm4,%xmm1");
+		asm volatile ("pxor %xmm4,%xmm2");
+		asm volatile ("pxor %xmm4,%xmm3");
+		asm volatile ("pxor %xmm4,%xmm6");
+
+		asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i]));
+		asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i]));
+		asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i]));
+		asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i]));
+		asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i]));
+	}
+
+	raid_sse_end();
 }
 
-void raid_gen4_ssse3_aes(int nd, size_t size, void **vv)
+/*
+ * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
+ */
+void raid_gen6_ssse3_raid(int nd, size_t size, void **vv)
 {
-	raid_gen4_ssse3_gen(nd, size, vv, 3);
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p;
+	uint8_t *q;
+	uint8_t *r;
+	uint8_t *s;
+	uint8_t *t;
+	uint8_t *u;
+	int d, l;
+	size_t i;
+	uint8_t buffer[2 * 16 + 16];
+	uint8_t *pd = __align_ptr(buffer, 16);
+
+	l = nd - 1;
+	p = v[nd];
+	q = v[nd + 1];
+	r = v[nd + 2];
+	s = v[nd + 3];
+	t = v[nd + 4];
+	u = v[nd + 5];
+
+	/* special case with only one data disk */
+	if (l == 0) {
+		for (i = 0; i < 6; ++i)
+			memcpy(v[1 + i], v[0], size);
+		return;
+	}
+
+	raid_sse_begin();
+
+	/* generic case with at least two data disks */
+	for (i = 0; i < size; i += 16) {
+		/* last disk without the generator multiplication */
+		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
+
+		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
+		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16]));
+
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+		asm volatile ("movdqa %xmm4,%xmm5");
+		asm volatile ("psrlw  $4,%xmm5");
+		asm volatile ("pand   %xmm7,%xmm4");
+		asm volatile ("pand   %xmm7,%xmm5");
+
+		asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm0");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm0");
+
+		asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm1");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm1");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm2");
+
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
+		asm volatile ("pshufb %xmm4,%xmm3");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pxor   %xmm7,%xmm3");
+
+		/* intermediate disks */
+		for (d = l - 1; d > 0; --d) {
+			asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
+
+			asm volatile ("pxor %xmm4,%xmm4");
+			asm volatile ("pcmpgtb %xmm6,%xmm4");
+			asm volatile ("paddb %xmm6,%xmm6");
+			asm volatile ("pand %xmm7,%xmm4");
+			asm volatile ("pxor %xmm4,%xmm6");
+
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
+
+			asm volatile ("pxor %xmm4,%xmm5");
+			asm volatile ("pxor %xmm4,%xmm6");
+			asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0]));
+			asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16]));
+
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+			asm volatile ("movdqa %xmm4,%xmm5");
+			asm volatile ("psrlw  $4,%xmm5");
+			asm volatile ("pand   %xmm7,%xmm4");
+			asm volatile ("pand   %xmm7,%xmm5");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm0");
+			asm volatile ("pxor   %xmm7,%xmm0");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm1");
+			asm volatile ("pxor   %xmm7,%xmm1");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm2");
+			asm volatile ("pxor   %xmm7,%xmm2");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pxor   %xmm6,%xmm3");
+			asm volatile ("pxor   %xmm7,%xmm3");
+		}
+
+		/* first disk with all coefficients at 1 */
+		asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
+		asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
+
+		asm volatile ("pxor %xmm4,%xmm4");
+		asm volatile ("pcmpgtb %xmm6,%xmm4");
+		asm volatile ("paddb %xmm6,%xmm6");
+		asm volatile ("pand %xmm7,%xmm4");
+		asm volatile ("pxor %xmm4,%xmm6");
+
+		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
+		asm volatile ("pxor %xmm4,%xmm0");
+		asm volatile ("pxor %xmm4,%xmm1");
+		asm volatile ("pxor %xmm4,%xmm2");
+		asm volatile ("pxor %xmm4,%xmm3");
+		asm volatile ("pxor %xmm4,%xmm5");
+		asm volatile ("pxor %xmm4,%xmm6");
+
+		asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i]));
+		asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i]));
+		asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i]));
+		asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i]));
+		asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i]));
+		asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i]));
+	}
+
+	raid_sse_end();
 }
 
 #ifdef CONFIG_X86_64
+/*
+ * GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
+ *
+ * Note that it uses 16 registers, meaning that x64 is required.
+ */
+static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p;
+	uint8_t *q;
+	uint8_t *r;
+	int d, l;
+	size_t i;
+
+	l = nd - 1;
+	p = v[nd];
+	q = v[nd + 1];
+	r = v[nd + 2];
+
+	/* special case with only one data disk */
+	if (l == 0) {
+		for (i = 0; i < 3; ++i)
+			memcpy(v[1 + i], v[0], size);
+		return;
+	}
+
+	raid_sse_begin();
+
+	/* generic case with at least two data disks */
+	asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));
+	asm volatile ("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 32) {
+		/* last disk without the generator multiplication */
+		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
+		asm volatile ("movdqa %0,%%xmm12" : : "m" (v[l][i + 16]));
+
+		asm volatile ("movdqa %xmm4,%xmm0");
+		asm volatile ("movdqa %xmm4,%xmm1");
+		asm volatile ("movdqa %xmm12,%xmm8");
+		asm volatile ("movdqa %xmm12,%xmm9");
+
+		asm volatile ("movdqa %xmm4,%xmm5");
+		asm volatile ("movdqa %xmm12,%xmm13");
+		asm volatile ("psrlw  $4,%xmm5");
+		asm volatile ("psrlw  $4,%xmm13");
+		asm volatile ("pand   %xmm11,%xmm4");
+		asm volatile ("pand   %xmm11,%xmm12");
+		asm volatile ("pand   %xmm11,%xmm5");
+		asm volatile ("pand   %xmm11,%xmm13");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
+		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
+		asm volatile ("movdqa %xmm2,%xmm10");
+		asm volatile ("movdqa %xmm7,%xmm15");
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm12,%xmm10");
+		asm volatile ("pshufb %xmm5,%xmm7");
+		asm volatile ("pshufb %xmm13,%xmm15");
+		asm volatile ("pxor   %xmm7,%xmm2");
+		asm volatile ("pxor   %xmm15,%xmm10");
+
+		/* process two intermediate disks per iteration */
+		for (d = l - 1; d > 1; d -= 2) {
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
+			asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
+
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
+			asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
+
+			if (generator == 3) {
+				asm volatile ("movdqa %xmm1,%xmm6");
+				asm volatile ("movdqa %xmm9,%xmm14");
+			}
+			asm volatile ("pxor %xmm5,%xmm5");
+			asm volatile ("pxor %xmm13,%xmm13");
+			asm volatile ("pcmpgtb %xmm1,%xmm5");
+			asm volatile ("pcmpgtb %xmm9,%xmm13");
+			asm volatile ("paddb %xmm1,%xmm1");
+			asm volatile ("paddb %xmm9,%xmm9");
+			asm volatile ("pand %xmm3,%xmm5");
+			asm volatile ("pand %xmm3,%xmm13");
+			asm volatile ("pxor %xmm5,%xmm1");
+			asm volatile ("pxor %xmm13,%xmm9");
+			if (generator == 3) {
+				asm volatile ("pxor %xmm6,%xmm1");
+				asm volatile ("pxor %xmm14,%xmm9");
+			}
+
+			asm volatile ("pxor %xmm4,%xmm0");
+			asm volatile ("pxor %xmm4,%xmm1");
+			asm volatile ("pxor %xmm12,%xmm8");
+			asm volatile ("pxor %xmm12,%xmm9");
+
+			asm volatile ("movdqa %xmm4,%xmm5");
+			asm volatile ("movdqa %xmm12,%xmm13");
+			asm volatile ("psrlw  $4,%xmm5");
+			asm volatile ("psrlw  $4,%xmm13");
+			asm volatile ("pand   %xmm11,%xmm4");
+			asm volatile ("pand   %xmm11,%xmm12");
+			asm volatile ("pand   %xmm11,%xmm5");
+			asm volatile ("pand   %xmm11,%xmm13");
+
+			asm volatile ("movdqa %xmm7,%xmm6");
+			asm volatile ("movdqa %xmm15,%xmm14");
+			asm volatile ("pshufb %xmm4,%xmm7");
+			asm volatile ("pshufb %xmm12,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm15");
+			asm volatile ("pshufb %xmm13,%xmm14");
+			asm volatile ("pxor   %xmm7,%xmm2");
+			asm volatile ("pxor   %xmm6,%xmm10");
+			asm volatile ("pxor   %xmm15,%xmm2");
+			asm volatile ("pxor   %xmm14,%xmm10");
+
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d - 1][i]));
+			asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d - 1][i + 16]));
+
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d - 1][1][0][0]));
+			asm volatile ("movdqa %0,%%xmm15" : : "m" (raid_gfcauchypshufb[d - 1][1][1][0]));
+
+			if (generator == 3) {
+				asm volatile ("movdqa %xmm1,%xmm6");
+				asm volatile ("movdqa %xmm9,%xmm14");
+			}
+			asm volatile ("pxor %xmm5,%xmm5");
+			asm volatile ("pxor %xmm13,%xmm13");
+			asm volatile ("pcmpgtb %xmm1,%xmm5");
+			asm volatile ("pcmpgtb %xmm9,%xmm13");
+			asm volatile ("paddb %xmm1,%xmm1");
+			asm volatile ("paddb %xmm9,%xmm9");
+			asm volatile ("pand %xmm3,%xmm5");
+			asm volatile ("pand %xmm3,%xmm13");
+			asm volatile ("pxor %xmm5,%xmm1");
+			asm volatile ("pxor %xmm13,%xmm9");
+			if (generator == 3) {
+				asm volatile ("pxor %xmm6,%xmm1");
+				asm volatile ("pxor %xmm14,%xmm9");
+			}
+
+			asm volatile ("pxor %xmm4,%xmm0");
+			asm volatile ("pxor %xmm4,%xmm1");
+			asm volatile ("pxor %xmm12,%xmm8");
+			asm volatile ("pxor %xmm12,%xmm9");
+
+			asm volatile ("movdqa %xmm4,%xmm5");
+			asm volatile ("movdqa %xmm12,%xmm13");
+			asm volatile ("psrlw  $4,%xmm5");
+			asm volatile ("psrlw  $4,%xmm13");
+			asm volatile ("pand   %xmm11,%xmm4");
+			asm volatile ("pand   %xmm11,%xmm12");
+			asm volatile ("pand   %xmm11,%xmm5");
+			asm volatile ("pand   %xmm11,%xmm13");
+
+			asm volatile ("movdqa %xmm7,%xmm6");
+			asm volatile ("movdqa %xmm15,%xmm14");
+			asm volatile ("pshufb %xmm4,%xmm7");
+			asm volatile ("pshufb %xmm12,%xmm6");
+			asm volatile ("pshufb %xmm5,%xmm15");
+			asm volatile ("pshufb %xmm13,%xmm14");
+			asm volatile ("pxor   %xmm7,%xmm2");
+			asm volatile ("pxor   %xmm6,%xmm10");
+			asm volatile ("pxor   %xmm15,%xmm2");
+			asm volatile ("pxor   %xmm14,%xmm10");
+		}
+
+		/* single remaining intermediate disk */
+		if (d == 1) {
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[1][i]));
+			asm volatile ("movdqa %0,%%xmm12" : : "m" (v[1][i + 16]));
+
+			if (generator == 3) {
+				asm volatile ("movdqa %xmm1,%xmm6");
+				asm volatile ("movdqa %xmm9,%xmm14");
+			}
+			asm volatile ("pxor %xmm5,%xmm5");
+			asm volatile ("pxor %xmm13,%xmm13");
+			asm volatile ("pcmpgtb %xmm1,%xmm5");
+			asm volatile ("pcmpgtb %xmm9,%xmm13");
+			asm volatile ("paddb %xmm1,%xmm1");
+			asm volatile ("paddb %xmm9,%xmm9");
+			asm volatile ("pand %xmm3,%xmm5");
+			asm volatile ("pand %xmm3,%xmm13");
+			asm volatile ("pxor %xmm5,%xmm1");
+			asm volatile ("pxor %xmm13,%xmm9");
+			if (generator == 3) {
+				asm volatile ("pxor %xmm6,%xmm1");
+				asm volatile ("pxor %xmm14,%xmm9");
+			}
+
+			asm volatile ("pxor %xmm4,%xmm0");
+			asm volatile ("pxor %xmm4,%xmm1");
+			asm volatile ("pxor %xmm12,%xmm8");
+			asm volatile ("pxor %xmm12,%xmm9");
+
+			asm volatile ("movdqa %xmm4,%xmm5");
+			asm volatile ("movdqa %xmm12,%xmm13");
+			asm volatile ("psrlw  $4,%xmm5");
+			asm volatile ("psrlw  $4,%xmm13");
+			asm volatile ("pand   %xmm11,%xmm4");
+			asm volatile ("pand   %xmm11,%xmm12");
+			asm volatile ("pand   %xmm11,%xmm5");
+			asm volatile ("pand   %xmm11,%xmm13");
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[1][1][0][0]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[1][1][1][0]));
+			asm volatile ("movdqa %xmm6,%xmm14");
+			asm volatile ("movdqa %xmm7,%xmm15");
+			asm volatile ("pshufb %xmm4,%xmm6");
+			asm volatile ("pshufb %xmm12,%xmm14");
+			asm volatile ("pshufb %xmm5,%xmm7");
+			asm volatile ("pshufb %xmm13,%xmm15");
+			asm volatile ("pxor   %xmm6,%xmm2");
+			asm volatile ("pxor   %xmm14,%xmm10");
+			asm volatile ("pxor   %xmm7,%xmm2");
+			asm volatile ("pxor   %xmm15,%xmm10");
+		}
+
+		/* first disk with all coefficients at 1 */
+		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
+		asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
+
+		if (generator == 3) {
+			asm volatile ("movdqa %xmm1,%xmm6");
+			asm volatile ("movdqa %xmm9,%xmm14");
+		}
+		asm volatile ("pxor %xmm5,%xmm5");
+		asm volatile ("pxor %xmm13,%xmm13");
+		asm volatile ("pcmpgtb %xmm1,%xmm5");
+		asm volatile ("pcmpgtb %xmm9,%xmm13");
+		asm volatile ("paddb %xmm1,%xmm1");
+		asm volatile ("paddb %xmm9,%xmm9");
+		asm volatile ("pand %xmm3,%xmm5");
+		asm volatile ("pand %xmm3,%xmm13");
+		asm volatile ("pxor %xmm5,%xmm1");
+		asm volatile ("pxor %xmm13,%xmm9");
+		if (generator == 3) {
+			asm volatile ("pxor %xmm6,%xmm1");
+			asm volatile ("pxor %xmm14,%xmm9");
+		}
+
+		asm volatile ("pxor %xmm4,%xmm0");
+		asm volatile ("pxor %xmm4,%xmm1");
+		asm volatile ("pxor %xmm4,%xmm2");
+		asm volatile ("pxor %xmm12,%xmm8");
+		asm volatile ("pxor %xmm12,%xmm9");
+		asm volatile ("pxor %xmm12,%xmm10");
+
+		asm volatile ("movntdq %%xmm0,%0" : "=m" (p[i]));
+		asm volatile ("movntdq %%xmm8,%0" : "=m" (p[i + 16]));
+		asm volatile ("movntdq %%xmm1,%0" : "=m" (q[i]));
+		asm volatile ("movntdq %%xmm9,%0" : "=m" (q[i + 16]));
+		asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
+		asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));
+	}
+
+	raid_sse_end();
+}
+
 /*
  * GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
  *
@@ -709,153 +968,6 @@ static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **v
 	raid_sse_end();
 }
 
-void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv)
-{
-	raid_gen4_ssse3ext_gen(nd, size, vv, 2);
-}
-
-void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv)
-{
-	raid_gen4_ssse3ext_gen(nd, size, vv, 3);
-}
-#endif
-
-/*
- * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
- */
-void raid_gen5_ssse3_raid(int nd, size_t size, void **vv)
-{
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p;
-	uint8_t *q;
-	uint8_t *r;
-	uint8_t *s;
-	uint8_t *t;
-	int d, l;
-	size_t i;
-	uint8_t buffer[16 + 16];
-	uint8_t *pd = __align_ptr(buffer, 16);
-
-	l = nd - 1;
-	p = v[nd];
-	q = v[nd + 1];
-	r = v[nd + 2];
-	s = v[nd + 3];
-	t = v[nd + 4];
-
-	/* special case with only one data disk */
-	if (l == 0) {
-		for (i = 0; i < 5; ++i)
-			memcpy(v[1 + i], v[0], size);
-		return;
-	}
-
-	raid_sse_begin();
-
-	/* generic case with at least two data disks */
-	for (i = 0; i < size; i += 16) {
-		/* last disk without the generator multiplication */
-		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
-
-		asm volatile ("movdqa %xmm4,%xmm0");
-		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
-
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-		asm volatile ("movdqa %xmm4,%xmm5");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("pand   %xmm7,%xmm4");
-		asm volatile ("pand   %xmm7,%xmm5");
-
-		asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm1");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm1");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm2");
-
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm3");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm3");
-
-		/* intermediate disks */
-		for (d = l - 1; d > 0; --d) {
-			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
-
-			asm volatile ("pxor %xmm5,%xmm5");
-			asm volatile ("pcmpgtb %xmm0,%xmm5");
-			asm volatile ("paddb %xmm0,%xmm0");
-			asm volatile ("pand %xmm7,%xmm5");
-			asm volatile ("pxor %xmm5,%xmm0");
-
-			asm volatile ("pxor %xmm4,%xmm0");
-			asm volatile ("pxor %xmm4,%xmm6");
-			asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[0]));
-
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-			asm volatile ("movdqa %xmm4,%xmm5");
-			asm volatile ("psrlw  $4,%xmm5");
-			asm volatile ("pand   %xmm7,%xmm4");
-			asm volatile ("pand   %xmm7,%xmm5");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm1");
-			asm volatile ("pxor   %xmm7,%xmm1");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm2");
-			asm volatile ("pxor   %xmm7,%xmm2");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm3");
-			asm volatile ("pxor   %xmm7,%xmm3");
-		}
-
-		/* first disk with all coefficients at 1 */
-		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
-		asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
-
-		asm volatile ("pxor %xmm5,%xmm5");
-		asm volatile ("pcmpgtb %xmm0,%xmm5");
-		asm volatile ("paddb %xmm0,%xmm0");
-		asm volatile ("pand %xmm7,%xmm5");
-		asm volatile ("pxor %xmm5,%xmm0");
-
-		asm volatile ("pxor %xmm4,%xmm0");
-		asm volatile ("pxor %xmm4,%xmm1");
-		asm volatile ("pxor %xmm4,%xmm2");
-		asm volatile ("pxor %xmm4,%xmm3");
-		asm volatile ("pxor %xmm4,%xmm6");
-
-		asm volatile ("movntdq %%xmm6,%0" : "=m" (p[i]));
-		asm volatile ("movntdq %%xmm0,%0" : "=m" (q[i]));
-		asm volatile ("movntdq %%xmm1,%0" : "=m" (r[i]));
-		asm volatile ("movntdq %%xmm2,%0" : "=m" (s[i]));
-		asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i]));
-	}
-
-	raid_sse_end();
-}
-
-#ifdef CONFIG_X86_64
 /*
  * GENX SSSE3ext implementation
  */
@@ -1035,199 +1147,12 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
 	raid_sse_end();
 }
 
-/*
- * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
- *
- * Note that it uses 16 registers, meaning that x64 is required.
- */
-void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv)
-{
-	raid_genX_ssse3ext(nd, size, vv, 5, 2);
-}
-
-void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv)
-{
-	raid_genX_ssse3ext(nd, size, vv, 5, 3);
-}
-#endif
-
-/*
- * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
- */
-void raid_gen6_ssse3_raid(int nd, size_t size, void **vv)
-{
-	uint8_t **v = (uint8_t **)vv;
-	uint8_t *p;
-	uint8_t *q;
-	uint8_t *r;
-	uint8_t *s;
-	uint8_t *t;
-	uint8_t *u;
-	int d, l;
-	size_t i;
-	uint8_t buffer[2 * 16 + 16];
-	uint8_t *pd = __align_ptr(buffer, 16);
-
-	l = nd - 1;
-	p = v[nd];
-	q = v[nd + 1];
-	r = v[nd + 2];
-	s = v[nd + 3];
-	t = v[nd + 4];
-	u = v[nd + 5];
-
-	/* special case with only one data disk */
-	if (l == 0) {
-		for (i = 0; i < 6; ++i)
-			memcpy(v[1 + i], v[0], size);
-		return;
-	}
-
-	raid_sse_begin();
-
-	/* generic case with at least two data disks */
-	for (i = 0; i < size; i += 16) {
-		/* last disk without the generator multiplication */
-		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[l][i]));
-
-		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[0]));
-		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd[16]));
-
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-		asm volatile ("movdqa %xmm4,%xmm5");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("pand   %xmm7,%xmm4");
-		asm volatile ("pand   %xmm7,%xmm5");
-
-		asm volatile ("movdqa %0,%%xmm0" : : "m" (raid_gfcauchypshufb[l][1][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][1][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm0");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm0");
-
-		asm volatile ("movdqa %0,%%xmm1" : : "m" (raid_gfcauchypshufb[l][2][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][2][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm1");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm1");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfcauchypshufb[l][3][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][3][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm2");
-
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfcauchypshufb[l][4][0][0]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[l][4][1][0]));
-		asm volatile ("pshufb %xmm4,%xmm3");
-		asm volatile ("pshufb %xmm5,%xmm7");
-		asm volatile ("pxor   %xmm7,%xmm3");
-
-		/* intermediate disks */
-		for (d = l - 1; d > 0; --d) {
-			asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
-
-			asm volatile ("pxor %xmm4,%xmm4");
-			asm volatile ("pcmpgtb %xmm6,%xmm4");
-			asm volatile ("paddb %xmm6,%xmm6");
-			asm volatile ("pand %xmm7,%xmm4");
-			asm volatile ("pxor %xmm4,%xmm6");
-
-			asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
-
-			asm volatile ("pxor %xmm4,%xmm5");
-			asm volatile ("pxor %xmm4,%xmm6");
-			asm volatile ("movdqa %%xmm5,%0" : "=m" (pd[0]));
-			asm volatile ("movdqa %%xmm6,%0" : "=m" (pd[16]));
-
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-			asm volatile ("movdqa %xmm4,%xmm5");
-			asm volatile ("psrlw  $4,%xmm5");
-			asm volatile ("pand   %xmm7,%xmm4");
-			asm volatile ("pand   %xmm7,%xmm5");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][1][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][1][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm0");
-			asm volatile ("pxor   %xmm7,%xmm0");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][2][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][2][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm1");
-			asm volatile ("pxor   %xmm7,%xmm1");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][3][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][3][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm2");
-			asm volatile ("pxor   %xmm7,%xmm2");
-
-			asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfcauchypshufb[d][4][0][0]));
-			asm volatile ("movdqa %0,%%xmm7" : : "m" (raid_gfcauchypshufb[d][4][1][0]));
-			asm volatile ("pshufb %xmm4,%xmm6");
-			asm volatile ("pshufb %xmm5,%xmm7");
-			asm volatile ("pxor   %xmm6,%xmm3");
-			asm volatile ("pxor   %xmm7,%xmm3");
-		}
-
-		/* first disk with all coefficients at 1 */
-		asm volatile ("movdqa %0,%%xmm5" : : "m" (pd[0]));
-		asm volatile ("movdqa %0,%%xmm6" : : "m" (pd[16]));
-		asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
-
-		asm volatile ("pxor %xmm4,%xmm4");
-		asm volatile ("pcmpgtb %xmm6,%xmm4");
-		asm volatile ("paddb %xmm6,%xmm6");
-		asm volatile ("pand %xmm7,%xmm4");
-		asm volatile ("pxor %xmm4,%xmm6");
-
-		asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
-		asm volatile ("pxor %xmm4,%xmm0");
-		asm volatile ("pxor %xmm4,%xmm1");
-		asm volatile ("pxor %xmm4,%xmm2");
-		asm volatile ("pxor %xmm4,%xmm3");
-		asm volatile ("pxor %xmm4,%xmm5");
-		asm volatile ("pxor %xmm4,%xmm6");
-
-		asm volatile ("movntdq %%xmm5,%0" : "=m" (p[i]));
-		asm volatile ("movntdq %%xmm6,%0" : "=m" (q[i]));
-		asm volatile ("movntdq %%xmm0,%0" : "=m" (r[i]));
-		asm volatile ("movntdq %%xmm1,%0" : "=m" (s[i]));
-		asm volatile ("movntdq %%xmm2,%0" : "=m" (t[i]));
-		asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i]));
-	}
-
-	raid_sse_end();
-}
-
-#ifdef CONFIG_X86_64
-/*
- * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
- *
- * Note that it uses 16 registers, meaning that x64 is required.
- */
-void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv)
-{
-	raid_genX_ssse3ext(nd, size, vv, 6, 2);
-}
-
-void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv)
-{
-	raid_genX_ssse3ext(nd, size, vv, 6, 3);
-}
 #endif
 
 /*
  * RAID recovering for one disk SSSE3 implementation
  */
-void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+static __always_inline void raid_rec1_ssse3_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
 	uint8_t **v = (uint8_t **)vv;
 	uint8_t *p;
@@ -1238,12 +1163,6 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	(void)nr; /* unused, it's always 1 */
 
-	/* if it's RAID5 uses the faster function */
-	if (ip[0] == 0) {
-		raid_rec1of1(id, nd, size, vv);
-		return;
-	}
-
 	/* setup the coefficients matrix */
 	G = A(ip[0], id[0]);
 
@@ -1258,7 +1177,503 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	raid_sse_begin();
 
+	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+	asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
+	asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
+
+	for (i = 0; i < size; i += 16) {
+		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
+		asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[i]));
+		asm volatile ("movdqa %xmm4,%xmm2");
+		asm volatile ("movdqa %xmm5,%xmm3");
+		asm volatile ("pxor %xmm0,%xmm1");
+		asm volatile ("movdqa %xmm1,%xmm0");
+		asm volatile ("psrlw $4,%xmm1");
+		asm volatile ("pand %xmm7,%xmm0");
+		asm volatile ("pand %xmm7,%xmm1");
+		asm volatile ("pshufb %xmm0,%xmm2");
+		asm volatile ("pshufb %xmm1,%xmm3");
+		asm volatile ("pxor %xmm3,%xmm2");
+		asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i]));
+	}
+
+	raid_sse_end();
+}
+
+/*
+ * RAID recovering for two disks SSSE3 implementation
+ */
+static __always_inline void raid_rec2_ssse3_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	const int N = 2;
+	uint8_t *p[N];
+	uint8_t *pa[N];
+	uint8_t G[N * N];
+	uint8_t V[N * N];
+	size_t i;
+	int j, k;
+
+	(void)nr; /* unused, it's always 2 */
+
+	/* setup the coefficients matrix */
+	for (j = 0; j < N; ++j)
+		for (k = 0; k < N; ++k)
+			G[j * N + k] = A(ip[j], id[k]);
+
+	/* invert it to solve the system of linear equations */
+	raid_invert(G, V, N);
+
+	/* compute delta parity */
+	raid_delta_gen(N, id, ip, nd, size, vv);
+
+	for (j = 0; j < N; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	raid_sse_begin();
+
+	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 16) {
+		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (pa[0][i]));
+		asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (pa[1][i]));
+		asm volatile ("pxor %xmm2,%xmm0");
+		asm volatile ("pxor %xmm3,%xmm1");
+
+		asm volatile ("pxor %xmm6,%xmm6");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
+		asm volatile ("movdqa %xmm0,%xmm4");
+		asm volatile ("movdqa %xmm0,%xmm5");
+		asm volatile ("psrlw $4,%xmm5");
+		asm volatile ("pand %xmm7,%xmm4");
+		asm volatile ("pand %xmm7,%xmm5");
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm5,%xmm3");
+		asm volatile ("pxor %xmm2,%xmm6");
+		asm volatile ("pxor %xmm3,%xmm6");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
+		asm volatile ("movdqa %xmm1,%xmm4");
+		asm volatile ("movdqa %xmm1,%xmm5");
+		asm volatile ("psrlw $4,%xmm5");
+		asm volatile ("pand %xmm7,%xmm4");
+		asm volatile ("pand %xmm7,%xmm5");
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm5,%xmm3");
+		asm volatile ("pxor %xmm2,%xmm6");
+		asm volatile ("pxor %xmm3,%xmm6");
+
+		asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[0][i]));
+
+		asm volatile ("pxor %xmm6,%xmm6");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
+		asm volatile ("movdqa %xmm0,%xmm4");
+		asm volatile ("movdqa %xmm0,%xmm5");
+		asm volatile ("psrlw $4,%xmm5");
+		asm volatile ("pand %xmm7,%xmm4");
+		asm volatile ("pand %xmm7,%xmm5");
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm5,%xmm3");
+		asm volatile ("pxor %xmm2,%xmm6");
+		asm volatile ("pxor %xmm3,%xmm6");
+
+		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
+		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
+		asm volatile ("movdqa %xmm1,%xmm4");
+		asm volatile ("movdqa %xmm1,%xmm5");
+		asm volatile ("psrlw $4,%xmm5");
+		asm volatile ("pand %xmm7,%xmm4");
+		asm volatile ("pand %xmm7,%xmm5");
+		asm volatile ("pshufb %xmm4,%xmm2");
+		asm volatile ("pshufb %xmm5,%xmm3");
+		asm volatile ("pxor %xmm2,%xmm6");
+		asm volatile ("pxor %xmm3,%xmm6");
+
+		asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));
+	}
+
+	raid_sse_end();
+}
+
+static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	uint8_t buffer_low[RAID_PARITY_MAX * 16 + 16];
+	uint8_t buffer_high[RAID_PARITY_MAX * 16 + 16];
+	uint8_t *pd_low = __align_ptr(buffer_low, 16);
+	uint8_t *pd_high = __align_ptr(buffer_high, 16);
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > 4);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_sse_begin();
+
+	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 16) {
+		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
+
+		if (nr >= 2)
+			asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
+
+		if (nr >= 3)
+			asm volatile ("movdqa %0,%%xmm2" : : "m" (p[2][i]));
+
+		if (nr >= 4)
+			asm volatile ("movdqa %0,%%xmm3" : : "m" (p[3][i]));
+
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (src[s][i]));
+
+			if (has_p) {
+				asm volatile ("pxor %xmm4,%xmm0");
+
+				asm volatile ("movdqa %xmm4,%xmm5");
+				asm volatile ("psrlw $4,%xmm5");
+				asm volatile ("pand %xmm7,%xmm4");
+				asm volatile ("pand %xmm7,%xmm5");
+			} else {
+				asm volatile ("movdqa %xmm4,%xmm5");
+				asm volatile ("psrlw $4,%xmm5");
+				asm volatile ("pand %xmm7,%xmm4");
+				asm volatile ("pand %xmm7,%xmm5");
+
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0][0]));
+				asm volatile ("pshufb %xmm4,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm0");
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[0][16]));
+				asm volatile ("pshufb %xmm5,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm0");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[1][0]));
+				asm volatile ("pshufb %xmm4,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm1");
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[1][16]));
+				asm volatile ("pshufb %xmm5,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm1");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[2][0]));
+				asm volatile ("pshufb %xmm4,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm2");
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[2][16]));
+				asm volatile ("pshufb %xmm5,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm2");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[3][0]));
+				asm volatile ("pshufb %xmm4,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm3");
+				asm volatile ("movdqa %0,%%xmm6" : : "m" (t[3][16]));
+				asm volatile ("pshufb %xmm5,%xmm6");
+				asm volatile ("pxor %xmm6,%xmm3");
+			}
+		}
+
+		/*
+		 * Split completed syndromes once and store the low/high nibbles.
+		 * The survivor scan is already complete.
+		 */
+
+		asm volatile ("movdqa %xmm0,%xmm4");
+		asm volatile ("movdqa %xmm0,%xmm5");
+		asm volatile ("psrlw $4,%xmm5");
+		asm volatile ("pand %xmm7,%xmm4");
+		asm volatile ("pand %xmm7,%xmm5");
+		asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[0]));
+		asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[0]));
+
+		if (nr >= 2) {
+			asm volatile ("movdqa %xmm1,%xmm4");
+			asm volatile ("movdqa %xmm1,%xmm5");
+			asm volatile ("psrlw $4,%xmm5");
+			asm volatile ("pand %xmm7,%xmm4");
+			asm volatile ("pand %xmm7,%xmm5");
+			asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[16]));
+			asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[16]));
+		}
+
+		if (nr >= 3) {
+			asm volatile ("movdqa %xmm2,%xmm4");
+			asm volatile ("movdqa %xmm2,%xmm5");
+			asm volatile ("psrlw $4,%xmm5");
+			asm volatile ("pand %xmm7,%xmm4");
+			asm volatile ("pand %xmm7,%xmm5");
+			asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[32]));
+			asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[32]));
+		}
+
+		if (nr >= 4) {
+			asm volatile ("movdqa %xmm3,%xmm4");
+			asm volatile ("movdqa %xmm3,%xmm5");
+			asm volatile ("psrlw $4,%xmm5");
+			asm volatile ("pand %xmm7,%xmm4");
+			asm volatile ("pand %xmm7,%xmm5");
+			asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[48]));
+			asm volatile ("movdqa %%xmm5,%0" : "=m" (pd_high[48]));
+		}
+
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("movdqa %0,%%xmm0" : : "m" (t[0][0]));
+			asm volatile ("movdqa %0,%%xmm1" : : "m" (t[0][16]));
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[0]));
+			asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[0]));
+			asm volatile ("pshufb %xmm4,%xmm0");
+			asm volatile ("pshufb %xmm5,%xmm1");
+
+			for (k = 1; k < nr; ++k) {
+				asm volatile ("movdqa %0,%%xmm2" : : "m" (t[k][0]));
+				asm volatile ("movdqa %0,%%xmm3" : : "m" (t[k][16]));
+				asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[k * 16]));
+				asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[k * 16]));
+				asm volatile ("pshufb %xmm4,%xmm2");
+				asm volatile ("pshufb %xmm5,%xmm3");
+				asm volatile ("pxor %xmm2,%xmm0");
+				asm volatile ("pxor %xmm3,%xmm1");
+			}
+
+			asm volatile ("pxor %xmm1,%xmm0");
+			asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_sse_end();
+}
+
+static __always_inline void raid_recX_ssse3(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	uint8_t buffer_low[RAID_PARITY_MAX * 16 + 16];
+	uint8_t buffer_high[RAID_PARITY_MAX * 16 + 16];
+	uint8_t *pd_low = __align_ptr(buffer_low, 16);
+	uint8_t *pd_high = __align_ptr(buffer_high, 16);
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_sse_begin();
+
+	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
+
+	for (i = 0; i < size; i += 16) {
+		/* pd_low[] temporarily contains the complete raw syndrome */
+		for (j = 0; j < nr; ++j) {
+			asm volatile ("movdqa %0,%%xmm0" : : "m" (p[j][i]));
+			asm volatile ("movdqa %%xmm0,%0" : "=m" (pd_low[j * 16]));
+		}
+
+		/* single survivor scan */
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("movdqa %0,%%xmm0" : : "m" (src[s][i]));
+
+			/* p must use the original source before nibble splitting */
+			if (has_p) {
+				asm volatile ("movdqa %xmm0,%xmm4");
+				asm volatile ("pxor %0,%%xmm4" : : "m" (pd_low[0]));
+				asm volatile ("movdqa %%xmm4,%0" : "=m" (pd_low[0]));
+			}
+
+			asm volatile ("movdqa %xmm0,%xmm1");
+			asm volatile ("psrlw $4,%xmm1");
+			asm volatile ("pand %xmm7,%xmm0");
+			asm volatile ("pand %xmm7,%xmm1");
+
+			for (j = has_p; j < nr; ++j) {
+				asm volatile ("movdqa %0,%%xmm2" : : "m" (t[j][0]));
+				asm volatile ("movdqa %0,%%xmm3" : : "m" (t[j][16]));
+				asm volatile ("pshufb %xmm0,%xmm2");
+				asm volatile ("pshufb %xmm1,%xmm3");
+				asm volatile ("pxor %xmm3,%xmm2");
+				asm volatile ("pxor %0,%%xmm2" : : "m" (pd_low[j * 16]));
+				asm volatile ("movdqa %%xmm2,%0" : "=m" (pd_low[j * 16]));
+			}
+		}
+
+		/* convert all completed raw syndromes to low/high nibble form */
+		for (k = 0; k < nr; ++k) {
+			asm volatile ("movdqa %0,%%xmm0" : : "m" (pd_low[k * 16]));
+			asm volatile ("movdqa %xmm0,%xmm1");
+			asm volatile ("psrlw $4,%xmm1");
+			asm volatile ("pand %xmm7,%xmm0");
+			asm volatile ("pand %xmm7,%xmm1");
+			asm volatile ("movdqa %%xmm0,%0" : "=m" (pd_low[k * 16]));
+			asm volatile ("movdqa %%xmm1,%0" : "=m" (pd_high[k * 16]));
+		}
+
+		/* reconstruct */
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("movdqa %0,%%xmm0" : : "m" (t[0][0]));
+			asm volatile ("movdqa %0,%%xmm1" : : "m" (t[0][16]));
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[0]));
+			asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[0]));
+			asm volatile ("pshufb %xmm4,%xmm0");
+			asm volatile ("pshufb %xmm5,%xmm1");
+
+			for (k = 1; k < nr; ++k) {
+				asm volatile ("movdqa %0,%%xmm2" : : "m" (t[k][0]));
+				asm volatile ("movdqa %0,%%xmm3" : : "m" (t[k][16]));
+				asm volatile ("movdqa %0,%%xmm4" : : "m" (pd_low[k * 16]));
+				asm volatile ("movdqa %0,%%xmm5" : : "m" (pd_high[k * 16]));
+				asm volatile ("pshufb %xmm4,%xmm2");
+				asm volatile ("pshufb %xmm5,%xmm3");
+				asm volatile ("pxor %xmm2,%xmm0");
+				asm volatile ("pxor %xmm3,%xmm1");
+			}
+
+			asm volatile ("pxor %xmm1,%xmm0");
+			asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_sse_end();
+}
+
 #ifdef CONFIG_X86_64
+/*
+ * RAID recovering for one disk SSSE3 extended implementation
+ */
+static __always_inline void raid_rec1_ssse3ext_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p;
+	uint8_t *pa;
+	uint8_t G;
+	uint8_t V;
+	size_t i;
+
+	(void)nr; /* unused, it's always 1 */
+
+	/* setup the coefficients matrix */
+	G = A(ip[0], id[0]);
+
+	/* invert it to solve the system of linear equations */
+	V = inv(G);
+
+	/* compute delta parity */
+	raid_delta_gen(1, id, ip, nd, size, vv);
+
+	p = v[nd + ip[0]];
+	pa = v[id[0]];
+
+	raid_sse_begin();
+
 	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
 	asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
 	asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
@@ -1329,35 +1744,14 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 		asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32]));
 		asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48]));
 	}
-#else /* CONFIG_X86_32 */
-	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-	asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
-	asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
-
-	for (i = 0; i < size; i += 16) {
-		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
-		asm volatile ("movdqa %0,%%xmm1" : : "m" (pa[i]));
-		asm volatile ("movdqa %xmm4,%xmm2");
-		asm volatile ("movdqa %xmm5,%xmm3");
-		asm volatile ("pxor   %xmm0,%xmm1");
-		asm volatile ("movdqa %xmm1,%xmm0");
-		asm volatile ("psrlw  $4,%xmm1");
-		asm volatile ("pand   %xmm7,%xmm0");
-		asm volatile ("pand   %xmm7,%xmm1");
-		asm volatile ("pshufb %xmm0,%xmm2");
-		asm volatile ("pshufb %xmm1,%xmm3");
-		asm volatile ("pxor   %xmm3,%xmm2");
-		asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i]));
-	}
-#endif
 
 	raid_sse_end();
 }
 
 /*
- * RAID recovering for two disks SSSE3 implementation
+ * RAID recovering for two disks SSSE3 extended implementation
  */
-void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+static __always_inline void raid_rec2_ssse3ext_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 {
 	uint8_t **v = (uint8_t **)vv;
 	const int N = 2;
@@ -1388,10 +1782,9 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 	raid_sse_begin();
 
-#ifdef CONFIG_X86_64
 	asm volatile ("movdqa %0,%%xmm6" : : "m" (gfconst16.low4[0]));
 
-	/* the inverse matrix V[] is constant for the whole recovery. */
+	/* the inverse matrix V[] is constant for the whole recovery */
 	asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
 	asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
 	asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
@@ -1429,9 +1822,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 		asm volatile ("pand %xmm6,%xmm1");
 		asm volatile ("pand %xmm6,%xmm3");
 
-		/*
-		 * pa[0] = V[0] * d0 ^ V[1] * d1
-		 */
+		/* pa[0] = V[0] * d0 ^ V[1] * d1 */
 		asm volatile ("movdqa %xmm8,%xmm4");
 		asm volatile ("pshufb %xmm0,%xmm4");
 		asm volatile ("movdqa %xmm9,%xmm5");
@@ -1447,9 +1838,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 		asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[0][i]));
 
-		/*
-		 * pa[1] = V[2] * d0 ^ V[3] * d1
-		 */
+		/* pa[1] = V[2] * d0 ^ V[3] * d1 */
 		asm volatile ("movdqa %xmm12,%xmm4");
 		asm volatile ("pshufb %xmm0,%xmm4");
 		asm volatile ("movdqa %xmm13,%xmm5");
@@ -1465,336 +1854,684 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
 
 		asm volatile ("movdqa %%xmm4,%0" : "=m" (pa[1][i]));
 	}
-#else /* CONFIG_X86_32 */
-	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-
-	for (i = 0; i < size; i += 16) {
-		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (pa[0][i]));
-		asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (pa[1][i]));
-		asm volatile ("pxor   %xmm2,%xmm0");
-		asm volatile ("pxor   %xmm3,%xmm1");
-
-		asm volatile ("pxor %xmm6,%xmm6");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
-		asm volatile ("movdqa %xmm0,%xmm4");
-		asm volatile ("movdqa %xmm0,%xmm5");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("pand   %xmm7,%xmm4");
-		asm volatile ("pand   %xmm7,%xmm5");
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm5,%xmm3");
-		asm volatile ("pxor   %xmm2,%xmm6");
-		asm volatile ("pxor   %xmm3,%xmm6");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
-		asm volatile ("movdqa %xmm1,%xmm4");
-		asm volatile ("movdqa %xmm1,%xmm5");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("pand   %xmm7,%xmm4");
-		asm volatile ("pand   %xmm7,%xmm5");
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm5,%xmm3");
-		asm volatile ("pxor   %xmm2,%xmm6");
-		asm volatile ("pxor   %xmm3,%xmm6");
-
-		asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[0][i]));
-
-		asm volatile ("pxor %xmm6,%xmm6");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
-		asm volatile ("movdqa %xmm0,%xmm4");
-		asm volatile ("movdqa %xmm0,%xmm5");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("pand   %xmm7,%xmm4");
-		asm volatile ("pand   %xmm7,%xmm5");
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm5,%xmm3");
-		asm volatile ("pxor   %xmm2,%xmm6");
-		asm volatile ("pxor   %xmm3,%xmm6");
-
-		asm volatile ("movdqa %0,%%xmm2" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
-		asm volatile ("movdqa %0,%%xmm3" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
-		asm volatile ("movdqa %xmm1,%xmm4");
-		asm volatile ("movdqa %xmm1,%xmm5");
-		asm volatile ("psrlw  $4,%xmm5");
-		asm volatile ("pand   %xmm7,%xmm4");
-		asm volatile ("pand   %xmm7,%xmm5");
-		asm volatile ("pshufb %xmm4,%xmm2");
-		asm volatile ("pshufb %xmm5,%xmm3");
-		asm volatile ("pxor   %xmm2,%xmm6");
-		asm volatile ("pxor   %xmm3,%xmm6");
-
-		asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));
-	}
-#endif
 
 	raid_sse_end();
 }
 
-/*
- * RAID recovering SSSE3 implementation
- */
-void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
 {
 	uint8_t **v = (uint8_t **)vv;
-	int N = nr;
 	uint8_t *p[RAID_PARITY_MAX];
 	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
 	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
 	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
 	size_t i;
-	int j, k;
+	int d, j, k, s;
+	int ns;
+	int has_p;
 
-	/* setup the coefficients matrix */
-	for (j = 0; j < N; ++j)
-		for (k = 0; k < N; ++k)
-			G[j * N + k] = A(ip[j], id[k]);
+	BUG_ON(nr < 1 || nr > 3);
 
-	/* invert it to solve the system of linear equations */
-	raid_invert(G, V, N);
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
 
-	/* compute delta parity */
-	raid_delta_gen(N, id, ip, nd, size, vv);
+	raid_invert(G, V, nr);
 
-	for (j = 0; j < N; ++j) {
+	for (j = 0; j < nr; ++j) {
 		p[j] = v[nd + ip[j]];
 		pa[j] = v[id[j]];
 	}
 
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
 	raid_sse_begin();
 
-#ifdef CONFIG_X86_64
-	const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	for (i = 0; i < size; i += 32) {
+		/*
+		 * Xmm15 is later reused by reconstruction, therefore reload the
+		 * nibble mask on every iteration.
+		 */
+		asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
 
-	/* precompute shuffle table pointers */
-	for (j = 0; j < N * N; ++j)
-		T[j] = &raid_gfmulpshufb[V[j]][0][0];
+		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
+		asm volatile ("movdqa %0,%%xmm1" : : "m" (p[0][i + 16]));
 
-	asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
-
-	for (i = 0; i < size; i += 16) {
-		/* delta */
-		asm volatile (
-			"movq 0(%2), %%rax\n"
-			"movq 0(%3), %%rbx\n"
-			"movdqa (%%rax, %1), %%xmm12\n"
-			"movdqa (%%rbx, %1), %%xmm13\n"
-			"pxor %%xmm13, %%xmm12\n"
-			"movdqa %%xmm12, %%xmm1\n"
-			"psrlw $4, %%xmm1\n"
-			"pand %%xmm15, %%xmm12\n"
-			"pand %%xmm15, %%xmm1\n"
-			"movdqa %%xmm12, %%xmm0\n"
-			"cmpq $1, %0\n"
-			"jbe 1f\n"
-
-			"movq 8(%2), %%rax\n"
-			"movq 8(%3), %%rbx\n"
-			"movdqa (%%rax, %1), %%xmm12\n"
-			"movdqa (%%rbx, %1), %%xmm13\n"
-			"pxor %%xmm13, %%xmm12\n"
-			"movdqa %%xmm12, %%xmm3\n"
-			"psrlw $4, %%xmm3\n"
-			"pand %%xmm15, %%xmm12\n"
-			"pand %%xmm15, %%xmm3\n"
-			"movdqa %%xmm12, %%xmm2\n"
-			"cmpq $2, %0\n"
-			"jbe 1f\n"
-
-			"movq 16(%2), %%rax\n"
-			"movq 16(%3), %%rbx\n"
-			"movdqa (%%rax, %1), %%xmm12\n"
-			"movdqa (%%rbx, %1), %%xmm13\n"
-			"pxor %%xmm13, %%xmm12\n"
-			"movdqa %%xmm12, %%xmm5\n"
-			"psrlw $4, %%xmm5\n"
-			"pand %%xmm15, %%xmm12\n"
-			"pand %%xmm15, %%xmm5\n"
-			"movdqa %%xmm12, %%xmm4\n"
-			"cmpq $3, %0\n"
-			"jbe 1f\n"
-
-			"movq 24(%2), %%rax\n"
-			"movq 24(%3), %%rbx\n"
-			"movdqa (%%rax, %1), %%xmm12\n"
-			"movdqa (%%rbx, %1), %%xmm13\n"
-			"pxor %%xmm13, %%xmm12\n"
-			"movdqa %%xmm12, %%xmm7\n"
-			"psrlw $4, %%xmm7\n"
-			"pand %%xmm15, %%xmm12\n"
-			"pand %%xmm15, %%xmm7\n"
-			"movdqa %%xmm12, %%xmm6\n"
-			"cmpq $4, %0\n"
-			"jbe 1f\n"
-
-			"movq 32(%2), %%rax\n"
-			"movq 32(%3), %%rbx\n"
-			"movdqa (%%rax, %1), %%xmm12\n"
-			"movdqa (%%rbx, %1), %%xmm13\n"
-			"pxor %%xmm13, %%xmm12\n"
-			"movdqa %%xmm12, %%xmm9\n"
-			"psrlw $4, %%xmm9\n"
-			"pand %%xmm15, %%xmm12\n"
-			"pand %%xmm15, %%xmm9\n"
-			"movdqa %%xmm12, %%xmm8\n"
-			"cmpq $5, %0\n"
-			"jbe 1f\n"
-
-			"movq 40(%2), %%rax\n"
-			"movq 40(%3), %%rbx\n"
-			"movdqa (%%rax, %1), %%xmm12\n"
-			"movdqa (%%rbx, %1), %%xmm13\n"
-			"pxor %%xmm13, %%xmm12\n"
-			"movdqa %%xmm12, %%xmm11\n"
-			"psrlw $4, %%xmm11\n"
-			"pand %%xmm15, %%xmm12\n"
-			"pand %%xmm15, %%xmm11\n"
-			"movdqa %%xmm12, %%xmm10\n"
-
-			"1:\n"
-			:
-			: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
-			: "rax", "rbx", "cc", "memory"
-		);
-
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"movq 0(%2), %%rcx\n"
-				"movdqa 0(%%rcx), %%xmm13\n"
-				"movdqa 16(%%rcx), %%xmm14\n"
-				"pshufb %%xmm0, %%xmm13\n"
-				"pshufb %%xmm1, %%xmm14\n"
-				"cmpq $1, %0\n"
-				"jbe 1f\n"
-
-				"movq 8(%2), %%rcx\n"
-				"movdqa 0(%%rcx), %%xmm12\n"
-				"pshufb %%xmm2, %%xmm12\n"
-				"pxor %%xmm12, %%xmm13\n"
-				"movdqa 16(%%rcx), %%xmm12\n"
-				"pshufb %%xmm3, %%xmm12\n"
-				"pxor %%xmm12, %%xmm14\n"
-				"cmpq $2, %0\n"
-				"jbe 1f\n"
-
-				"movq 16(%2), %%rcx\n"
-				"movdqa 0(%%rcx), %%xmm12\n"
-				"pshufb %%xmm4, %%xmm12\n"
-				"pxor %%xmm12, %%xmm13\n"
-				"movdqa 16(%%rcx), %%xmm12\n"
-				"pshufb %%xmm5, %%xmm12\n"
-				"pxor %%xmm12, %%xmm14\n"
-				"cmpq $3, %0\n"
-				"jbe 1f\n"
-
-				"movq 24(%2), %%rcx\n"
-				"movdqa 0(%%rcx), %%xmm12\n"
-				"pshufb %%xmm6, %%xmm12\n"
-				"pxor %%xmm12, %%xmm13\n"
-				"movdqa 16(%%rcx), %%xmm12\n"
-				"pshufb %%xmm7, %%xmm12\n"
-				"pxor %%xmm12, %%xmm14\n"
-				"cmpq $4, %0\n"
-				"jbe 1f\n"
-
-				"movq 32(%2), %%rcx\n"
-				"movdqa 0(%%rcx), %%xmm12\n"
-				"pshufb %%xmm8, %%xmm12\n"
-				"pxor %%xmm12, %%xmm13\n"
-				"movdqa 16(%%rcx), %%xmm12\n"
-				"pshufb %%xmm9, %%xmm12\n"
-				"pxor %%xmm12, %%xmm14\n"
-				"cmpq $5, %0\n"
-				"jbe 1f\n"
-
-				"movq 40(%2), %%rcx\n"
-				"movdqa 0(%%rcx), %%xmm12\n"
-				"pshufb %%xmm10, %%xmm12\n"
-				"pxor %%xmm12, %%xmm13\n"
-				"movdqa 16(%%rcx), %%xmm12\n"
-				"pshufb %%xmm11, %%xmm12\n"
-				"pxor %%xmm12, %%xmm14\n"
-
-				"1:\n"
-				"pxor %%xmm14, %%xmm13\n"
-				"movq %3, %%rax\n"
-				"movdqa %%xmm13, (%%rax, %1)\n"
-				:
-				: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
-				: "rax", "rcx", "cc", "memory"
-			);
-		}
-	}
-#else /* CONFIG_X86_32 */
-	uint8_t buffer_low[RAID_PARITY_MAX * 16 + 16];
-	uint8_t buffer_high[RAID_PARITY_MAX * 16 + 16];
-	uint8_t *pd_low = __align_ptr(buffer_low, 16);
-	uint8_t *pd_high = __align_ptr(buffer_high, 16);
-
-	asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
-
-	for (i = 0; i < size; i += 16) {
-		/* delta */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"movdqa %2, %%xmm0\n"
-				"movdqa %3, %%xmm1\n"
-				"pxor   %%xmm1, %%xmm0\n"
-				"movdqa %%xmm0, %%xmm1\n"
-				"psrlw  $4, %%xmm1\n"
-				"pand   %%xmm7, %%xmm0\n"
-				"pand   %%xmm7, %%xmm1\n"
-				"movdqa %%xmm0, %0\n"
-				"movdqa %%xmm1, %1\n"
-				: "=m" (pd_low[j * 16]), "=m" (pd_high[j * 16])
-				: "m" (p[j][i]), "m" (pa[j][i])
-			);
+		if (nr >= 2) {
+			asm volatile ("movdqa %0,%%xmm2" : : "m" (p[1][i]));
+			asm volatile ("movdqa %0,%%xmm3" : : "m" (p[1][i + 16]));
 		}
 
-		/* reconstruct */
-		for (j = 0; j < N; ++j) {
-			asm volatile (
-				"pxor %%xmm0, %%xmm0\n"
-				"pxor %%xmm1, %%xmm1\n"
-			);
+		if (nr >= 3) {
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (p[2][i]));
+			asm volatile ("movdqa %0,%%xmm5" : : "m" (p[2][i + 16]));
+		}
 
-			for (k = 0; k < N; ++k) {
-				uint8_t m = V[j * N + k];
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
 
-				asm volatile (
-					"movdqa %0, %%xmm2\n"
-					"movdqa %1, %%xmm3\n"
-					"movdqa %2, %%xmm4\n"
-					"movdqa %3, %%xmm5\n"
-					"pshufb %%xmm4, %%xmm2\n"
-					"pshufb %%xmm5, %%xmm3\n"
-					"pxor   %%xmm2, %%xmm0\n"
-					"pxor   %%xmm3, %%xmm1\n"
-					:
-					: "m" (raid_gfmulpshufb[m][0][0]), "m" (raid_gfmulpshufb[m][1][0]),
-					"m" (pd_low[k * 16]), "m" (pd_high[k * 16])
-				);
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (src[s][i]));
+			asm volatile ("movdqa %0,%%xmm7" : : "m" (src[s][i + 16]));
+
+			if (has_p) {
+				asm volatile ("pxor %xmm6,%xmm0");
+				asm volatile ("pxor %xmm7,%xmm1");
+
+				asm volatile ("movdqa %xmm6,%xmm8");
+				asm volatile ("movdqa %xmm7,%xmm9");
+				asm volatile ("psrlw $4,%xmm8");
+				asm volatile ("psrlw $4,%xmm9");
+				asm volatile ("pand %xmm15,%xmm6");
+				asm volatile ("pand %xmm15,%xmm7");
+				asm volatile ("pand %xmm15,%xmm8");
+				asm volatile ("pand %xmm15,%xmm9");
+			} else {
+				asm volatile ("movdqa %xmm6,%xmm8");
+				asm volatile ("movdqa %xmm7,%xmm9");
+				asm volatile ("psrlw $4,%xmm8");
+				asm volatile ("psrlw $4,%xmm9");
+				asm volatile ("pand %xmm15,%xmm6");
+				asm volatile ("pand %xmm15,%xmm7");
+				asm volatile ("pand %xmm15,%xmm8");
+				asm volatile ("pand %xmm15,%xmm9");
+
+				asm volatile ("movdqa %0,%%xmm10" : : "m" (t[0][0]));
+				asm volatile ("movdqa %0,%%xmm11" : : "m" (t[0][16]));
+				asm volatile ("movdqa %xmm10,%xmm12");
+				asm volatile ("movdqa %xmm11,%xmm13");
+				asm volatile ("pshufb %xmm6,%xmm10");
+				asm volatile ("pshufb %xmm8,%xmm11");
+				asm volatile ("pxor %xmm11,%xmm10");
+				asm volatile ("pxor %xmm10,%xmm0");
+				asm volatile ("pshufb %xmm7,%xmm12");
+				asm volatile ("pshufb %xmm9,%xmm13");
+				asm volatile ("pxor %xmm13,%xmm12");
+				asm volatile ("pxor %xmm12,%xmm1");
 			}
 
-			asm volatile (
-				"pxor   %%xmm1, %%xmm0\n"
-				"movdqa %%xmm0, %0\n"
-				: "=m" (pa[j][i])
-			);
+			if (nr >= 2) {
+				asm volatile ("movdqa %0,%%xmm10" : : "m" (t[1][0]));
+				asm volatile ("movdqa %0,%%xmm11" : : "m" (t[1][16]));
+				asm volatile ("movdqa %xmm10,%xmm12");
+				asm volatile ("movdqa %xmm11,%xmm13");
+				asm volatile ("pshufb %xmm6,%xmm10");
+				asm volatile ("pshufb %xmm8,%xmm11");
+				asm volatile ("pxor %xmm11,%xmm10");
+				asm volatile ("pxor %xmm10,%xmm2");
+				asm volatile ("pshufb %xmm7,%xmm12");
+				asm volatile ("pshufb %xmm9,%xmm13");
+				asm volatile ("pxor %xmm13,%xmm12");
+				asm volatile ("pxor %xmm12,%xmm3");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("movdqa %0,%%xmm10" : : "m" (t[2][0]));
+				asm volatile ("movdqa %0,%%xmm11" : : "m" (t[2][16]));
+				asm volatile ("movdqa %xmm10,%xmm12");
+				asm volatile ("movdqa %xmm11,%xmm13");
+				asm volatile ("pshufb %xmm6,%xmm10");
+				asm volatile ("pshufb %xmm8,%xmm11");
+				asm volatile ("pxor %xmm11,%xmm10");
+				asm volatile ("pxor %xmm10,%xmm4");
+				asm volatile ("pshufb %xmm7,%xmm12");
+				asm volatile ("pshufb %xmm9,%xmm13");
+				asm volatile ("pxor %xmm13,%xmm12");
+				asm volatile ("pxor %xmm12,%xmm5");
+			}
+		}
+
+		if (nr >= 3) {
+			asm volatile ("movdqa %xmm4,%xmm8");
+			asm volatile ("movdqa %xmm4,%xmm9");
+			asm volatile ("psrlw $4,%xmm9");
+			asm volatile ("pand %xmm15,%xmm8");
+			asm volatile ("pand %xmm15,%xmm9");
+
+			asm volatile ("movdqa %xmm5,%xmm10");
+			asm volatile ("movdqa %xmm5,%xmm11");
+			asm volatile ("psrlw $4,%xmm11");
+			asm volatile ("pand %xmm15,%xmm10");
+			asm volatile ("pand %xmm15,%xmm11");
+		}
+
+		if (nr >= 2) {
+			asm volatile ("movdqa %xmm2,%xmm4");
+			asm volatile ("movdqa %xmm2,%xmm5");
+			asm volatile ("psrlw $4,%xmm5");
+			asm volatile ("pand %xmm15,%xmm4");
+			asm volatile ("pand %xmm15,%xmm5");
+
+			asm volatile ("movdqa %xmm3,%xmm6");
+			asm volatile ("movdqa %xmm3,%xmm7");
+			asm volatile ("psrlw $4,%xmm7");
+			asm volatile ("pand %xmm15,%xmm6");
+			asm volatile ("pand %xmm15,%xmm7");
+		}
+
+		asm volatile ("movdqa %xmm1,%xmm2");
+		asm volatile ("movdqa %xmm1,%xmm3");
+		asm volatile ("psrlw $4,%xmm3");
+		asm volatile ("pand %xmm15,%xmm2");
+		asm volatile ("pand %xmm15,%xmm3");
+
+		asm volatile ("movdqa %xmm0,%xmm1");
+		asm volatile ("psrlw $4,%xmm1");
+		asm volatile ("pand %xmm15,%xmm0");
+		asm volatile ("pand %xmm15,%xmm1");
+
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("pxor %xmm12,%xmm12");
+			asm volatile ("pxor %xmm13,%xmm13");
+
+			/* coefficient 0 */
+			asm volatile ("movdqa %0,%%xmm14" : : "m" (t[0][0]));
+			asm volatile ("movdqa %xmm14,%xmm15");
+			asm volatile ("pshufb %xmm0,%xmm14");
+			asm volatile ("pshufb %xmm2,%xmm15");
+			asm volatile ("pxor %xmm14,%xmm12");
+			asm volatile ("pxor %xmm15,%xmm13");
+
+			asm volatile ("movdqa %0,%%xmm14" : : "m" (t[0][16]));
+			asm volatile ("movdqa %xmm14,%xmm15");
+			asm volatile ("pshufb %xmm1,%xmm14");
+			asm volatile ("pshufb %xmm3,%xmm15");
+			asm volatile ("pxor %xmm14,%xmm12");
+			asm volatile ("pxor %xmm15,%xmm13");
+
+			if (nr >= 2) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][0]));
+				asm volatile ("movdqa %xmm14,%xmm15");
+				asm volatile ("pshufb %xmm4,%xmm14");
+				asm volatile ("pshufb %xmm6,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][16]));
+				asm volatile ("movdqa %xmm14,%xmm15");
+				asm volatile ("pshufb %xmm5,%xmm14");
+				asm volatile ("pshufb %xmm7,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][0]));
+				asm volatile ("movdqa %xmm14,%xmm15");
+				asm volatile ("pshufb %xmm8,%xmm14");
+				asm volatile ("pshufb %xmm10,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][16]));
+				asm volatile ("movdqa %xmm14,%xmm15");
+				asm volatile ("pshufb %xmm9,%xmm14");
+				asm volatile ("pshufb %xmm11,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			asm volatile ("movdqa %%xmm12,%0" : "=m" (pa[j][i]));
+			asm volatile ("movdqa %%xmm13,%0" : "=m" (pa[j][i + 16]));
 		}
 	}
-#endif
 
 	raid_sse_end();
 }
 
+static __always_inline void raid_recX_ssse3ext(int nr, int *id, int *ip,
+	int nd, size_t size, void **vv)
+{
+	uint8_t **v = (uint8_t **)vv;
+	uint8_t *p[RAID_PARITY_MAX];
+	uint8_t *pa[RAID_PARITY_MAX];
+	uint8_t *src[RAID_DATA_MAX];
+	uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
+	const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
+	const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
+	size_t i;
+	int d, j, k, s;
+	int ns;
+	int has_p;
+
+	BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			G[j * nr + k] = A(ip[j], id[k]);
+
+	raid_invert(G, V, nr);
+
+	for (j = 0; j < nr; ++j) {
+		p[j] = v[nd + ip[j]];
+		pa[j] = v[id[j]];
+	}
+
+	has_p = ip[0] == 0;
+
+	ns = 0;
+	k = 0;
+
+	for (d = 0; d < nd; ++d) {
+		if (k < nr && d == id[k]) {
+			++k;
+			continue;
+		}
+
+		src[ns] = v[d];
+
+		for (j = 0; j < nr; ++j)
+			S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
+
+		++ns;
+	}
+
+	BUG_ON(k != nr);
+	BUG_ON(ns != nd - nr);
+
+	for (j = 0; j < nr; ++j)
+		for (k = 0; k < nr; ++k)
+			R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
+
+	raid_sse_begin();
+
+	for (i = 0; i < size; i += 16) {
+		/*
+		 * Xmm15 is later reused by reconstruction, therefore reload the
+		 * nibble mask on every iteration.
+		 */
+		asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0]));
+
+		asm volatile ("movdqa %0,%%xmm0" : : "m" (p[0][i]));
+
+		if (nr >= 2)
+			asm volatile ("movdqa %0,%%xmm1" : : "m" (p[1][i]));
+
+		if (nr >= 3)
+			asm volatile ("movdqa %0,%%xmm2" : : "m" (p[2][i]));
+
+		if (nr >= 4)
+			asm volatile ("movdqa %0,%%xmm3" : : "m" (p[3][i]));
+
+		if (nr >= 5)
+			asm volatile ("movdqa %0,%%xmm4" : : "m" (p[4][i]));
+
+		if (nr >= 6)
+			asm volatile ("movdqa %0,%%xmm5" : : "m" (p[5][i]));
+
+		for (s = 0; s < ns; ++s) {
+			const uint8_t **t = S[s];
+
+			asm volatile ("movdqa %0,%%xmm6" : : "m" (src[s][i]));
+
+			if (has_p) {
+				asm volatile ("pxor %xmm6,%xmm0");
+
+				asm volatile ("movdqa %xmm6,%xmm7");
+				asm volatile ("psrlw $4,%xmm7");
+				asm volatile ("pand %xmm15,%xmm6");
+				asm volatile ("pand %xmm15,%xmm7");
+			} else {
+				asm volatile ("movdqa %xmm6,%xmm7");
+				asm volatile ("psrlw $4,%xmm7");
+				asm volatile ("pand %xmm15,%xmm6");
+				asm volatile ("pand %xmm15,%xmm7");
+
+				asm volatile ("movdqa %0,%%xmm8" : : "m" (t[0][0]));
+				asm volatile ("movdqa %0,%%xmm9" : : "m" (t[0][16]));
+				asm volatile ("pshufb %xmm6,%xmm8");
+				asm volatile ("pshufb %xmm7,%xmm9");
+				asm volatile ("pxor %xmm8,%xmm0");
+				asm volatile ("pxor %xmm9,%xmm0");
+			}
+
+			if (nr >= 2) {
+				asm volatile ("movdqa %0,%%xmm8" : : "m" (t[1][0]));
+				asm volatile ("movdqa %0,%%xmm9" : : "m" (t[1][16]));
+				asm volatile ("pshufb %xmm6,%xmm8");
+				asm volatile ("pshufb %xmm7,%xmm9");
+				asm volatile ("pxor %xmm8,%xmm1");
+				asm volatile ("pxor %xmm9,%xmm1");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("movdqa %0,%%xmm8" : : "m" (t[2][0]));
+				asm volatile ("movdqa %0,%%xmm9" : : "m" (t[2][16]));
+				asm volatile ("pshufb %xmm6,%xmm8");
+				asm volatile ("pshufb %xmm7,%xmm9");
+				asm volatile ("pxor %xmm8,%xmm2");
+				asm volatile ("pxor %xmm9,%xmm2");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("movdqa %0,%%xmm8" : : "m" (t[3][0]));
+				asm volatile ("movdqa %0,%%xmm9" : : "m" (t[3][16]));
+				asm volatile ("pshufb %xmm6,%xmm8");
+				asm volatile ("pshufb %xmm7,%xmm9");
+				asm volatile ("pxor %xmm8,%xmm3");
+				asm volatile ("pxor %xmm9,%xmm3");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("movdqa %0,%%xmm8" : : "m" (t[4][0]));
+				asm volatile ("movdqa %0,%%xmm9" : : "m" (t[4][16]));
+				asm volatile ("pshufb %xmm6,%xmm8");
+				asm volatile ("pshufb %xmm7,%xmm9");
+				asm volatile ("pxor %xmm8,%xmm4");
+				asm volatile ("pxor %xmm9,%xmm4");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("movdqa %0,%%xmm8" : : "m" (t[5][0]));
+				asm volatile ("movdqa %0,%%xmm9" : : "m" (t[5][16]));
+				asm volatile ("pshufb %xmm6,%xmm8");
+				asm volatile ("pshufb %xmm7,%xmm9");
+				asm volatile ("pxor %xmm8,%xmm5");
+				asm volatile ("pxor %xmm9,%xmm5");
+			}
+		}
+
+		/* expand backwards to low/high pairs */
+		if (nr >= 6) {
+			asm volatile ("movdqa %xmm5,%xmm10");
+			asm volatile ("movdqa %xmm5,%xmm11");
+			asm volatile ("psrlw $4,%xmm11");
+			asm volatile ("pand %xmm15,%xmm10");
+			asm volatile ("pand %xmm15,%xmm11");
+		}
+
+		if (nr >= 5) {
+			asm volatile ("movdqa %xmm4,%xmm8");
+			asm volatile ("movdqa %xmm4,%xmm9");
+			asm volatile ("psrlw $4,%xmm9");
+			asm volatile ("pand %xmm15,%xmm8");
+			asm volatile ("pand %xmm15,%xmm9");
+		}
+
+		if (nr >= 4) {
+			asm volatile ("movdqa %xmm3,%xmm6");
+			asm volatile ("movdqa %xmm3,%xmm7");
+			asm volatile ("psrlw $4,%xmm7");
+			asm volatile ("pand %xmm15,%xmm6");
+			asm volatile ("pand %xmm15,%xmm7");
+		}
+
+		if (nr >= 3) {
+			asm volatile ("movdqa %xmm2,%xmm4");
+			asm volatile ("movdqa %xmm2,%xmm5");
+			asm volatile ("psrlw $4,%xmm5");
+			asm volatile ("pand %xmm15,%xmm4");
+			asm volatile ("pand %xmm15,%xmm5");
+		}
+
+		if (nr >= 2) {
+			asm volatile ("movdqa %xmm1,%xmm2");
+			asm volatile ("movdqa %xmm1,%xmm3");
+			asm volatile ("psrlw $4,%xmm3");
+			asm volatile ("pand %xmm15,%xmm2");
+			asm volatile ("pand %xmm15,%xmm3");
+		}
+
+		asm volatile ("movdqa %xmm0,%xmm1");
+		asm volatile ("psrlw $4,%xmm1");
+		asm volatile ("pand %xmm15,%xmm0");
+		asm volatile ("pand %xmm15,%xmm1");
+
+		for (j = 0; j < nr; ++j) {
+			const uint8_t **t = R[j];
+
+			asm volatile ("movdqa %0,%%xmm12" : : "m" (t[0][0]));
+			asm volatile ("movdqa %0,%%xmm13" : : "m" (t[0][16]));
+			asm volatile ("pshufb %xmm0,%xmm12");
+			asm volatile ("pshufb %xmm1,%xmm13");
+
+			if (nr >= 2) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[1][0]));
+				asm volatile ("movdqa %0,%%xmm15" : : "m" (t[1][16]));
+				asm volatile ("pshufb %xmm2,%xmm14");
+				asm volatile ("pshufb %xmm3,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			if (nr >= 3) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[2][0]));
+				asm volatile ("movdqa %0,%%xmm15" : : "m" (t[2][16]));
+				asm volatile ("pshufb %xmm4,%xmm14");
+				asm volatile ("pshufb %xmm5,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			if (nr >= 4) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[3][0]));
+				asm volatile ("movdqa %0,%%xmm15" : : "m" (t[3][16]));
+				asm volatile ("pshufb %xmm6,%xmm14");
+				asm volatile ("pshufb %xmm7,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			if (nr >= 5) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[4][0]));
+				asm volatile ("movdqa %0,%%xmm15" : : "m" (t[4][16]));
+				asm volatile ("pshufb %xmm8,%xmm14");
+				asm volatile ("pshufb %xmm9,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			if (nr >= 6) {
+				asm volatile ("movdqa %0,%%xmm14" : : "m" (t[5][0]));
+				asm volatile ("movdqa %0,%%xmm15" : : "m" (t[5][16]));
+				asm volatile ("pshufb %xmm10,%xmm14");
+				asm volatile ("pshufb %xmm11,%xmm15");
+				asm volatile ("pxor %xmm14,%xmm12");
+				asm volatile ("pxor %xmm15,%xmm13");
+			}
+
+			asm volatile ("pxor %xmm13,%xmm12");
+			asm volatile ("movdqa %%xmm12,%0" : "=m" (pa[j][i]));
+		}
+	}
+
+	raid_sse_end();
+}
+
+#endif
+
+void raid_gen3_ssse3_raid(int nd, size_t size, void **vv)
+{
+	raid_gen3_ssse3_gen(nd, size, vv, 2);
+}
+
+void raid_gen3_ssse3_aes(int nd, size_t size, void **vv)
+{
+	raid_gen3_ssse3_gen(nd, size, vv, 3);
+}
+
+void raid_gen4_ssse3_raid(int nd, size_t size, void **vv)
+{
+	raid_gen4_ssse3_gen(nd, size, vv, 2);
+}
+
+void raid_gen4_ssse3_aes(int nd, size_t size, void **vv)
+{
+	raid_gen4_ssse3_gen(nd, size, vv, 3);
+}
+
+#ifdef CONFIG_X86_64
+void raid_gen3_ssse3ext_raid(int nd, size_t size, void **vv)
+{
+	raid_gen3_ssse3ext_gen(nd, size, vv, 2);
+}
+
+void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv)
+{
+	raid_gen3_ssse3ext_gen(nd, size, vv, 3);
+}
+
+void raid_gen4_ssse3ext_raid(int nd, size_t size, void **vv)
+{
+	raid_gen4_ssse3ext_gen(nd, size, vv, 2);
+}
+
+void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv)
+{
+	raid_gen4_ssse3ext_gen(nd, size, vv, 3);
+}
+
+/*
+ * GEN5 (penta parity with Cauchy matrix) SSSE3 implementation
+ *
+ * Note that it uses 16 registers, meaning that x64 is required.
+ */
+void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv)
+{
+	raid_genX_ssse3ext(nd, size, vv, 5, 2);
+}
+
+void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv)
+{
+	raid_genX_ssse3ext(nd, size, vv, 5, 3);
+}
+
+/*
+ * GEN6 (hexa parity with Cauchy matrix) SSSE3 implementation
+ *
+ * Note that it uses 16 registers, meaning that x64 is required.
+ */
+void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv)
+{
+	raid_genX_ssse3ext(nd, size, vv, 6, 2);
+}
+
+void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv)
+{
+	raid_genX_ssse3ext(nd, size, vv, 6, 3);
+}
+
+#endif
+
+void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 1);
+
+	/* if recovering with P uses the delta function */
+	if (ip[0] == 0) {
+		raid_rec1of1(id, nd, size, vv);
+		return;
+	}
+
+	raid_rec1_ssse3_delta(1, id, ip, nd, size, vv);
+}
+
+void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 2);
+	raid_rec2_ssse3_delta(2, id, ip, nd, size, vv);
+}
+
+void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 3);
+	raid_recX_ssse3_1234(3, id, ip, nd, size, vv);
+}
+
+void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_ssse3_1234(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_ssse3(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_ssse3(6, id, ip, nd, size, vv);
+}
+
+#ifdef CONFIG_X86_64
+void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 1);
+
+	/* if recovering with P uses the delta function */
+	if (ip[0] == 0) {
+		raid_rec1of1(id, nd, size, vv);
+		return;
+	}
+
+	raid_rec1_ssse3ext_delta(1, id, ip, nd, size, vv);
+}
+
+void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 2);
+	raid_rec2_ssse3ext_delta(2, id, ip, nd, size, vv);
+}
+
+void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 3);
+	raid_recX_ssse3ext_123(3, id, ip, nd, size, vv);
+}
+
+void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 4);
+	raid_recX_ssse3ext(4, id, ip, nd, size, vv);
+}
+
+void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 5);
+	raid_recX_ssse3ext(5, id, ip, nd, size, vv);
+}
+
+void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
+{
+	BUG_ON(nr != 6);
+	raid_recX_ssse3ext(6, id, ip, nd, size, vv);
+}
+
+#endif
+
 void raid_register_ssse3(void)
 {
 	if (raid_cpu_has_ssse3()) {
@@ -1804,6 +2541,14 @@ void raid_register_ssse3(void)
 		raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3_aes, RAID_POLY_AES);
 		raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3_raid, RAID_POLY_RAID);
 		raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3_raid, RAID_POLY_RAID);
+
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_rec3_ssse3, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_rec4_ssse3, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_rec5_ssse3, RAID_POLY_ANY);
+		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_rec6_ssse3, RAID_POLY_ANY);
+
 #ifdef CONFIG_X86_64
 		if (!raid_cpu_has_slow_extendedreg()) {
 			raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext_raid, RAID_POLY_RAID);
@@ -1814,15 +2559,16 @@ void raid_register_ssse3(void)
 			raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext_aes, RAID_POLY_AES);
 			raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext_raid, RAID_POLY_RAID);
 			raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext_aes, RAID_POLY_AES);
+
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3e", raid_rec1_ssse3ext, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3e", raid_rec2_ssse3ext, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_rec3_ssse3ext, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_rec4_ssse3ext, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_rec5_ssse3ext, RAID_POLY_ANY);
+			raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_rec6_ssse3ext, RAID_POLY_ANY);
 		}
 #endif
-
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "ssse3", raid_rec1_ssse3, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "ssse3", raid_rec2_ssse3, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
-		raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_recX_ssse3, RAID_POLY_ANY);
 	}
 }
+
 #endif
diff --git a/raid/test.c b/raid/test.c
index 89f2919f..ef4e907b 100644
--- a/raid/test.c
+++ b/raid/test.c
@@ -70,6 +70,19 @@ int raid_test_gfaffine(void)
 			if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x])
 				return -1;
 
+#ifdef CONFIG_X86_64
+	int d, p;
+
+	for (d = 0; d < RAID_DATA_MAX; ++d) {
+		for (p = 1; p < RAID_PARITY_MAX; ++p) {
+			if (memcmp(raid_gfcauchyaffine_raid[d][p - 1],
+				raid_gfaffine_raid[raid_gfcauchy_raid[p][d]],
+				8) != 0)
+				return -1;
+		}
+	}
+#endif
+
 	return 0;
 }
 
@@ -391,101 +404,245 @@ int raid_test_rec(int mode, int nd, size_t size)
 
 	/* setup recov functions */
 	for (i = 1; i <= np; ++i) {
-		if (i == 1) {
+		switch (i) {
+		case 1:
 			test_setup(i) = raid_rec1_int8;
-#ifdef CONFIG_X86
-			if (raid_cpu_has_ssse3())
-				test_setup(i) = raid_rec1_ssse3;
-			if (raid_cpu_has_avx2())
-				test_setup(i) = raid_rec1_avx2;
-#ifdef CONFIG_X86_64
-			if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
-				if (raid_cpu_has_avx2gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec1_avx2gfni_aes;
-					else
-						test_setup(i) = raid_rec1_avx2gfni_raid;
-				}
-				if (raid_cpu_has_avx512gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec1_avx512gfni_aes;
-					else
-						test_setup(i) = raid_rec1_avx512gfni_raid;
-				}
-			}
-#endif
-#endif
-#ifdef CONFIG_NEON
-			test_setup(i) = raid_rec1_neon;
-#endif
-#ifdef CONFIG_NEON32
-			test_setup(i) = raid_rec1_neon32;
-#endif
-		} else if (i == 2) {
+			break;
+		case 2:
 			test_setup(i) = raid_rec2_int8;
-#ifdef CONFIG_X86
-			if (raid_cpu_has_ssse3())
-				test_setup(i) = raid_rec2_ssse3;
-			if (raid_cpu_has_avx2())
-				test_setup(i) = raid_rec2_avx2;
-#ifdef CONFIG_X86_64
-			if (raid_cpu_has_avx512bw())
-				test_setup(i) = raid_rec2_avx512bw;
-			if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
-				if (raid_cpu_has_avx2gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec2_avx2gfni_aes;
-					else
-						test_setup(i) = raid_rec2_avx2gfni_raid;
-				}
-				if (raid_cpu_has_avx512gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec2_avx512gfni_aes;
-					else
-						test_setup(i) = raid_rec2_avx512gfni_raid;
-				}
-			}
-#endif
-#endif
-#ifdef CONFIG_NEON
-			test_setup(i) = raid_rec2_neon;
-#endif
-#ifdef CONFIG_NEON32
-			test_setup(i) = raid_rec2_neon32;
-#endif
-		} else {
+			break;
+		default:
 			test_setup(i) = raid_recX_int8;
+			break;
+		}
 #ifdef CONFIG_X86
-			if (raid_cpu_has_ssse3())
-				test_setup(i) = raid_recX_ssse3;
-			if (raid_cpu_has_avx2())
-				test_setup(i) = raid_recX_avx2;
-#ifdef CONFIG_X86_64
-			if (raid_cpu_has_avx512bw())
-				test_setup(i) = raid_recX_avx512bw;
-			if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
-				if (raid_cpu_has_avx2gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_recX_avx2gfni_aes;
-					else
-						test_setup(i) = raid_recX_avx2gfni_raid;
-				}
-				if (raid_cpu_has_avx512gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_recX_avx512gfni_aes;
-					else
-						test_setup(i) = raid_recX_avx512gfni_raid;
-				}
+		if (raid_cpu_has_ssse3()) {
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_ssse3;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_ssse3;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_ssse3;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_ssse3;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_ssse3;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_ssse3;
+				break;
+			}
+#ifdef CONFIG_X86_64
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_ssse3ext;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_ssse3ext;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_ssse3ext;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_ssse3ext;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_ssse3ext;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_ssse3ext;
+				break;
 			}
-#endif
-#endif
-#ifdef CONFIG_NEON
-			test_setup(i) = raid_recX_neon;
-#endif
-#ifdef CONFIG_NEON32
-			test_setup(i) = raid_recX_neon32;
 #endif
 		}
+		if (raid_cpu_has_avx2()) {
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_avx2;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_avx2;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_avx2;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_avx2;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_avx2;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_avx2;
+				break;
+			}
+		}
+#ifdef CONFIG_X86_64
+		if (raid_cpu_has_avx512bw()) {
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_avx512bw;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_avx512bw;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_avx512bw;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_avx512bw;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_avx512bw;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_avx512bw;
+				break;
+			}
+		}
+		if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
+			if (raid_cpu_has_avx2gfni()) {
+				if (mode == RAID_MODE_CAUCHY_AES) {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx2gfni_aes;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx2gfni_aes;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx2gfni_aes;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx2gfni_aes;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx2gfni_aes;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx2gfni_aes;
+						break;
+					}
+				} else {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx2gfni_raid;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx2gfni_raid;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx2gfni_raid;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx2gfni_raid;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx2gfni_raid;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx2gfni_raid;
+						break;
+					}
+				}
+			}
+			if (raid_cpu_has_avx512gfni()) {
+				if (mode == RAID_MODE_CAUCHY_AES) {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx512gfni_aes;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx512gfni_aes;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx512gfni_aes;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx512gfni_aes;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx512gfni_aes;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx512gfni_aes;
+						break;
+					}
+				} else {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx512gfni_raid;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx512gfni_raid;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx512gfni_raid;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx512gfni_raid;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx512gfni_raid;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx512gfni_raid;
+						break;
+					}
+				}
+			}
+		}
+#endif
+#endif
+#ifdef CONFIG_NEON
+		switch (i) {
+		case 1:
+			test_setup(i) = raid_rec1_neon;
+			break;
+		case 2:
+			test_setup(i) = raid_rec2_neon;
+			break;
+		case 3:
+			test_setup(i) = raid_rec3_neon;
+			break;
+		case 4:
+			test_setup(i) = raid_rec4_neon;
+			break;
+		case 5:
+			test_setup(i) = raid_rec5_neon;
+			break;
+		case 6:
+			test_setup(i) = raid_rec6_neon;
+			break;
+		}
+#endif
+#ifdef CONFIG_NEON32
+		switch (i) {
+		case 1:
+			test_setup(i) = raid_rec1_neon32;
+			break;
+		case 2:
+			test_setup(i) = raid_rec2_neon32;
+			break;
+		case 3:
+			test_setup(i) = raid_rec3_neon32;
+			break;
+		case 4:
+			test_setup(i) = raid_rec4_neon32;
+			break;
+		case 5:
+			test_setup(i) = raid_rec5_neon32;
+			break;
+		case 6:
+			test_setup(i) = raid_rec6_neon32;
+			break;
+		}
+#endif
 	}
 
 	/* compute the parity */
@@ -607,101 +764,245 @@ int raid_test_tail(int mode, int nd_max, size_t size)
 
 	/* setup recov functions */
 	for (i = 1; i <= np; ++i) {
-		if (i == 1) {
+		switch (i) {
+		case 1:
 			test_setup(i) = raid_rec1_int8;
-#ifdef CONFIG_X86
-			if (raid_cpu_has_ssse3())
-				test_setup(i) = raid_rec1_ssse3;
-			if (raid_cpu_has_avx2())
-				test_setup(i) = raid_rec1_avx2;
-#ifdef CONFIG_X86_64
-			if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
-				if (raid_cpu_has_avx2gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec1_avx2gfni_aes;
-					else
-						test_setup(i) = raid_rec1_avx2gfni_raid;
-				}
-				if (raid_cpu_has_avx512gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec1_avx512gfni_aes;
-					else
-						test_setup(i) = raid_rec1_avx512gfni_raid;
-				}
-			}
-#endif
-#endif
-#ifdef CONFIG_NEON
-			test_setup(i) = raid_rec1_neon;
-#endif
-#ifdef CONFIG_NEON32
-			test_setup(i) = raid_rec1_neon32;
-#endif
-		} else if (i == 2) {
+			break;
+		case 2:
 			test_setup(i) = raid_rec2_int8;
-#ifdef CONFIG_X86
-			if (raid_cpu_has_ssse3())
-				test_setup(i) = raid_rec2_ssse3;
-			if (raid_cpu_has_avx2())
-				test_setup(i) = raid_rec2_avx2;
-#ifdef CONFIG_X86_64
-			if (raid_cpu_has_avx512bw())
-				test_setup(i) = raid_rec2_avx512bw;
-			if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
-				if (raid_cpu_has_avx2gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec2_avx2gfni_aes;
-					else
-						test_setup(i) = raid_rec2_avx2gfni_raid;
-				}
-				if (raid_cpu_has_avx512gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_rec2_avx512gfni_aes;
-					else
-						test_setup(i) = raid_rec2_avx512gfni_raid;
-				}
-			}
-#endif
-#endif
-#ifdef CONFIG_NEON
-			test_setup(i) = raid_rec2_neon;
-#endif
-#ifdef CONFIG_NEON32
-			test_setup(i) = raid_rec2_neon32;
-#endif
-		} else {
+			break;
+		default:
 			test_setup(i) = raid_recX_int8;
+			break;
+		}
 #ifdef CONFIG_X86
-			if (raid_cpu_has_ssse3())
-				test_setup(i) = raid_recX_ssse3;
-			if (raid_cpu_has_avx2())
-				test_setup(i) = raid_recX_avx2;
-#ifdef CONFIG_X86_64
-			if (raid_cpu_has_avx512bw())
-				test_setup(i) = raid_recX_avx512bw;
-			if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
-				if (raid_cpu_has_avx2gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_recX_avx2gfni_aes;
-					else
-						test_setup(i) = raid_recX_avx2gfni_raid;
-				}
-				if (raid_cpu_has_avx512gfni()) {
-					if (mode == RAID_MODE_CAUCHY_AES)
-						test_setup(i) = raid_recX_avx512gfni_aes;
-					else
-						test_setup(i) = raid_recX_avx512gfni_raid;
-				}
+		if (raid_cpu_has_ssse3()) {
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_ssse3;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_ssse3;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_ssse3;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_ssse3;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_ssse3;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_ssse3;
+				break;
+			}
+#ifdef CONFIG_X86_64
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_ssse3ext;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_ssse3ext;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_ssse3ext;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_ssse3ext;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_ssse3ext;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_ssse3ext;
+				break;
 			}
-#endif
-#endif
-#ifdef CONFIG_NEON
-			test_setup(i) = raid_recX_neon;
-#endif
-#ifdef CONFIG_NEON32
-			test_setup(i) = raid_recX_neon32;
 #endif
 		}
+		if (raid_cpu_has_avx2()) {
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_avx2;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_avx2;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_avx2;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_avx2;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_avx2;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_avx2;
+				break;
+			}
+		}
+#ifdef CONFIG_X86_64
+		if (raid_cpu_has_avx512bw()) {
+			switch (i) {
+			case 1:
+				test_setup(i) = raid_rec1_avx512bw;
+				break;
+			case 2:
+				test_setup(i) = raid_rec2_avx512bw;
+				break;
+			case 3:
+				test_setup(i) = raid_rec3_avx512bw;
+				break;
+			case 4:
+				test_setup(i) = raid_rec4_avx512bw;
+				break;
+			case 5:
+				test_setup(i) = raid_rec5_avx512bw;
+				break;
+			case 6:
+				test_setup(i) = raid_rec6_avx512bw;
+				break;
+			}
+		}
+		if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
+			if (raid_cpu_has_avx2gfni()) {
+				if (mode == RAID_MODE_CAUCHY_AES) {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx2gfni_aes;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx2gfni_aes;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx2gfni_aes;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx2gfni_aes;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx2gfni_aes;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx2gfni_aes;
+						break;
+					}
+				} else {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx2gfni_raid;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx2gfni_raid;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx2gfni_raid;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx2gfni_raid;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx2gfni_raid;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx2gfni_raid;
+						break;
+					}
+				}
+			}
+			if (raid_cpu_has_avx512gfni()) {
+				if (mode == RAID_MODE_CAUCHY_AES) {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx512gfni_aes;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx512gfni_aes;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx512gfni_aes;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx512gfni_aes;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx512gfni_aes;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx512gfni_aes;
+						break;
+					}
+				} else {
+					switch (i) {
+					case 1:
+						test_setup(i) = raid_rec1_avx512gfni_raid;
+						break;
+					case 2:
+						test_setup(i) = raid_rec2_avx512gfni_raid;
+						break;
+					case 3:
+						test_setup(i) = raid_rec3_avx512gfni_raid;
+						break;
+					case 4:
+						test_setup(i) = raid_rec4_avx512gfni_raid;
+						break;
+					case 5:
+						test_setup(i) = raid_rec5_avx512gfni_raid;
+						break;
+					case 6:
+						test_setup(i) = raid_rec6_avx512gfni_raid;
+						break;
+					}
+				}
+			}
+		}
+#endif
+#endif
+#ifdef CONFIG_NEON
+		switch (i) {
+		case 1:
+			test_setup(i) = raid_rec1_neon;
+			break;
+		case 2:
+			test_setup(i) = raid_rec2_neon;
+			break;
+		case 3:
+			test_setup(i) = raid_rec3_neon;
+			break;
+		case 4:
+			test_setup(i) = raid_rec4_neon;
+			break;
+		case 5:
+			test_setup(i) = raid_rec5_neon;
+			break;
+		case 6:
+			test_setup(i) = raid_rec6_neon;
+			break;
+		}
+#endif
+#ifdef CONFIG_NEON32
+		switch (i) {
+		case 1:
+			test_setup(i) = raid_rec1_neon32;
+			break;
+		case 2:
+			test_setup(i) = raid_rec2_neon32;
+			break;
+		case 3:
+			test_setup(i) = raid_rec3_neon32;
+			break;
+		case 4:
+			test_setup(i) = raid_rec4_neon32;
+			break;
+		case 5:
+			test_setup(i) = raid_rec5_neon32;
+			break;
+		case 6:
+			test_setup(i) = raid_rec6_neon32;
+			break;
+		}
+#endif
 	}
 
 	/* test all disk counts from 1 to nd_max */
diff --git a/raid/test/Makefile b/raid/test/Makefile
index 29c51cb1..c4f29947 100644
--- a/raid/test/Makefile
+++ b/raid/test/Makefile
@@ -103,6 +103,11 @@ sdecovtest:
 
 # Use this target to run a complete test on Ice Lake simulator
 sdetest:
+	$(SDE) -p4p -- ./selftest
+	$(SDE) -mrm -- ./selftest
+	$(SDE) -nhm -- ./selftest
+	$(SDE) -hsw -- ./selftest
+	$(SDE) -skx -- ./selftest
 	$(SDE) -icl -- ./selftest
 	$(SDE) -icl -- ./fulltest
 
diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c
index 591ff39a..19b6137f 100644
--- a/raid/test/speedtest.c
+++ b/raid/test/speedtest.c
@@ -1122,10 +1122,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	printf("%8s", "neon32");
 #endif
 #ifdef CONFIG_X86
-	if (raid_cpu_has_ssse3())
+	if (raid_cpu_has_ssse3()) {
 		printf("%8s", "ssse3");
-	if (raid_cpu_has_avx2())
+#ifdef CONFIG_X86_64
+		printf("%8s", "ssse3e");
+#endif
+	}
+	if (raid_cpu_has_avx2()) {
 		printf("%8s", "avx2");
+#ifdef CONFIG_X86_64
+		printf("%8s", "avx2e");
+#endif
+	}
 #ifdef CONFIG_X86_64
 	if (raid_cpu_has_avx512bw())
 		printf("%8s", "avx512");
@@ -1172,12 +1180,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
+			/* raid_rec1of1() internally calls raid_gen() */
 			raid_gen_force(1, raid_gen1_sse2);
 			raid_rec1_ssse3(1, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* raid_rec1of1() internally calls raid_gen() */
+			raid_gen_force(1, raid_gen1_sse2);
+			raid_rec1_ssse3ext(1, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
@@ -1189,6 +1209,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/*
+			 * raid_rec1_avx2ext() uses raid_rec1of1() for P,
+			 * which internally calls raid_gen().
+			 */
+			raid_gen_force(1, raid_gen1_avx2);
+			raid_rec1_avx2ext(1, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
 			raid_gen_force(1, raid_gen1_avx512bw);
@@ -1236,7 +1269,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(1, raid_gen1_neon);
 		/* +1 to avoid GEN1 optimized case */
 		raid_rec1_neon(1, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1246,7 +1278,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(1, raid_gen1_neon32);
 		/* +1 to avoid GEN1 optimized case */
 		raid_rec1_neon32(1, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1258,6 +1289,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
+			/* ensure to use same hardware in the delta step */
 			raid_gen_force(1, raid_gen1_sse2);
 			/* +1 to avoid GEN1 optimized case */
 			raid_rec1_ssse3(1, id, ip + 1, nd, size, v);
@@ -1265,6 +1297,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_sse2);
+			/* +1 to avoid GEN1 optimized case */
+			raid_rec1_ssse3ext(1, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
@@ -1277,10 +1321,20 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_avx2);
+			/* +1 to avoid GEN1 optimized case */
+			raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(1, raid_gen1_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
+			/* +1 to avoid RAID5 optimized case */
 			raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
@@ -1289,8 +1343,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(1, raid_gen1_avx2);
-			/* +1 to avoid GEN1 optimized case */
+			/* +1 to avoid RAID5 optimized case */
 			raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
@@ -1299,8 +1352,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
-			/* +1 to avoid GEN1 optimized case */
+			/* +1 to avoid RAID5 optimized case */
 			raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
@@ -1311,6 +1363,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 	printf("\n");
 
+	/* REC1OF3 */
+	printf("%8s", "rec1of3");
+	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1));
+	fflush(stdout);
+
+	SPEED_START {
+		raid_gen_force(1, sizeof(void *) == 8 ? raid_gen1_int64 : raid_gen1_int32);
+		/* +2 to select R instead of Q */
+		raid_rec1_int8(1, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+
+#ifdef CONFIG_NEON
+	SPEED_START {
+		/* +2 to select R instead of Q */
+		raid_rec1_neon(1, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+#ifdef CONFIG_NEON32
+	SPEED_START {
+		/* +2 to select R instead of Q */
+		raid_rec1_neon32(1, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+
+#ifdef CONFIG_X86
+	if (raid_cpu_has_ssse3()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_sse2);
+			/* +2 to select R instead of Q */
+			raid_rec1_ssse3(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_sse2);
+			/* +2 to select R instead of Q */
+			raid_rec1_ssse3ext(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
+	}
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_gen_force(1, raid_gen1_avx2);
+			/* +2 to select R instead of Q */
+			raid_rec1_avx2(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(1, raid_gen1_avx2);
+			/* +2 to select R instead of Q */
+			raid_rec1_avx2ext(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
+			/* +2 to select R instead of Q */
+			raid_rec1_avx512bw(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx2gfni()) {
+		SPEED_START {
+			/* +2 to select R instead of Q */
+			raid_rec1_avx2gfni_raid(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512gfni()) {
+		SPEED_START {
+			/* +2 to select R instead of Q */
+			raid_rec1_avx512gfni_raid(1, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#endif
+#endif
+	printf("\n");
+
 	/* REC2OF2 */
 	printf("%8s", "rec2of2");
 	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
@@ -1326,7 +1488,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(2, raid_gen2_neon_raid);
 		raid_rec2_neon(2, id, ip, nd, size, v);
 	} SPEED_STOP
 
@@ -1335,7 +1496,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(2, raid_gen2_neon32_raid);
 		raid_rec2_neon32(2, id, ip, nd, size, v);
 	} SPEED_STOP
 
@@ -1346,24 +1506,28 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_sse2ext_raid);
-#else
+			/* ensure to use same hardware in the delta step */
 			raid_gen_force(2, raid_gen2_sse2_raid);
-#endif
 			raid_rec2_ssse3(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2ext_raid);
+			raid_rec2_ssse3ext(2, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_avx2ext_raid);
-#else
 			raid_gen_force(2, raid_gen2_avx2_raid);
-#endif
 			raid_rec2_avx2(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1371,9 +1535,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_avx2ext_raid);
+			raid_rec2_avx2ext(2, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(2, raid_gen2_avx512bw);
 			raid_rec2_avx512bw(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1382,7 +1555,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(2, raid_gen2_avx2gfni_raid);
 			raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1391,7 +1563,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(2, raid_gen2_avx512gfni_raid);
 			raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
 		} SPEED_STOP
 
@@ -1418,7 +1589,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(2, raid_gen2_neon_raid);
 		/* +1 to avoid GEN2 optimized case */
 		raid_rec2_neon(2, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1428,7 +1598,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(2, raid_gen2_neon32_raid);
 		/* +1 to avoid GEN2 optimized case */
 		raid_rec2_neon32(2, id, ip + 1, nd, size, v);
 	} SPEED_STOP
@@ -1440,25 +1609,30 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_sse2ext_raid);
-#else
+			/* ensure to use same hardware in the delta step */
 			raid_gen_force(2, raid_gen2_sse2_raid);
-#endif
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_ssse3(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2ext_raid);
+			/* +1 to avoid GEN2 optimized case */
+			raid_rec2_ssse3ext(2, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(2, raid_gen2_avx2ext_raid);
-#else
 			raid_gen_force(2, raid_gen2_avx2_raid);
-#endif
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx2(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1467,9 +1641,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_avx2ext_raid);
+			/* +1 to avoid GEN2 optimized case */
+			raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(2, raid_gen2_avx512bw);
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1479,7 +1663,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(2, raid_gen2_avx2gfni_raid);
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1489,7 +1672,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(2, raid_gen2_avx512gfni_raid);
 			/* +1 to avoid GEN2 optimized case */
 			raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
 		} SPEED_STOP
@@ -1501,6 +1683,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 	printf("\n");
 
+	/* REC2OF4 */
+	printf("%8s", "rec2of4");
+	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
+	fflush(stdout);
+
+	SPEED_START {
+		raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
+		/* +2 to select R,S instead of Q,R */
+		raid_rec2_int8(2, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+
+#ifdef CONFIG_NEON
+	SPEED_START {
+		/* +2 to select R,S instead of Q,R */
+		raid_rec2_neon(2, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+#ifdef CONFIG_NEON32
+	SPEED_START {
+		/* +2 to select R,S instead of Q,R */
+		raid_rec2_neon32(2, id, ip + 2, nd, size, v);
+	} SPEED_STOP
+
+	printf("%8" PRIu64, ds / dt);
+	fflush(stdout);
+#endif
+
+#ifdef CONFIG_X86
+	if (raid_cpu_has_ssse3()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_ssse3(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_sse2ext_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_ssse3ext(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
+	}
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_gen_force(2, raid_gen2_avx2_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx2(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			/* ensure to use same hardware in the delta step */
+			raid_gen_force(2, raid_gen2_avx2ext_raid);
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx2ext(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512bw()) {
+		SPEED_START {
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx512bw(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx2gfni()) {
+		SPEED_START {
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx2gfni_raid(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+	if (raid_cpu_has_avx512gfni()) {
+		SPEED_START {
+			/* +2 to select R,S instead of Q,R */
+			raid_rec2_avx512gfni_raid(2, id, ip + 2, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
+#endif
+#endif
+	printf("\n");
+
 	/* REC3OF3 */
 	printf("%8s", "rec3of3");
 	printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3));
@@ -1516,8 +1808,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(3, raid_gen3_neon_raid);
-		raid_recX_neon(3, id, ip, nd, size, v);
+		raid_rec3_neon(3, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1525,8 +1816,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(3, raid_gen3_neon32_raid);
-		raid_recX_neon32(3, id, ip, nd, size, v);
+		raid_rec3_neon32(3, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1536,36 +1826,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(3, raid_gen3_ssse3ext_raid);
-#else
-			raid_gen_force(3, raid_gen3_ssse3_raid);
-#endif
-			raid_recX_ssse3(3, id, ip, nd, size, v);
+			raid_rec3_ssse3(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec3_ssse3ext(3, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(3, raid_gen3_avx2ext_raid);
-#else
-			raid_gen_force(3, raid_gen3_ssse3_raid);
-#endif
-			raid_recX_avx2(3, id, ip, nd, size, v);
+			raid_rec3_avx2(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec3_avx2ext(3, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(3, raid_gen3_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(3, id, ip, nd, size, v);
+			raid_rec3_avx512bw(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1573,8 +1868,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(3, raid_gen3_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
+			raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1582,8 +1876,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(3, raid_gen3_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
+			raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1608,8 +1901,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(4, raid_gen4_neon_raid);
-		raid_recX_neon(4, id, ip, nd, size, v);
+		raid_rec4_neon(4, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1617,8 +1909,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(4, raid_gen4_neon32_raid);
-		raid_recX_neon32(4, id, ip, nd, size, v);
+		raid_rec4_neon32(4, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1628,36 +1919,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(4, raid_gen4_ssse3ext_raid);
-#else
-			raid_gen_force(4, raid_gen4_ssse3_raid);
-#endif
-			raid_recX_ssse3(4, id, ip, nd, size, v);
+			raid_rec4_ssse3(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec4_ssse3ext(4, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(4, raid_gen4_avx2ext_raid);
-#else
-			raid_gen_force(4, raid_gen4_ssse3_raid);
-#endif
-			raid_recX_avx2(4, id, ip, nd, size, v);
+			raid_rec4_avx2(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec4_avx2ext(4, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(4, raid_gen4_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(4, id, ip, nd, size, v);
+			raid_rec4_avx512bw(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1665,8 +1961,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(4, raid_gen4_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
+			raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1674,8 +1969,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(4, raid_gen4_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
+			raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1700,8 +1994,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(5, raid_gen5_neon_raid);
-		raid_recX_neon(5, id, ip, nd, size, v);
+		raid_rec5_neon(5, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1709,8 +2002,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(5, raid_gen5_neon32_raid);
-		raid_recX_neon32(5, id, ip, nd, size, v);
+		raid_rec5_neon32(5, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1720,36 +2012,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(5, raid_gen5_ssse3ext_raid);
-#else
-			raid_gen_force(5, raid_gen5_ssse3_raid);
-#endif
-			raid_recX_ssse3(5, id, ip, nd, size, v);
+			raid_rec5_ssse3(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec5_ssse3ext(5, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(5, raid_gen5_avx2ext_raid);
-#else
-			raid_gen_force(5, raid_gen5_ssse3_raid);
-#endif
-			raid_recX_avx2(5, id, ip, nd, size, v);
+			raid_rec5_avx2(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec5_avx2ext(5, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(5, raid_gen5_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(5, id, ip, nd, size, v);
+			raid_rec5_avx512bw(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1757,8 +2054,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(5, raid_gen5_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
+			raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1766,8 +2062,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(5, raid_gen5_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
+			raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1792,8 +2087,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 
 #ifdef CONFIG_NEON
 	SPEED_START {
-		raid_gen_force(6, raid_gen6_neon_raid);
-		raid_recX_neon(6, id, ip, nd, size, v);
+		raid_rec6_neon(6, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1801,8 +2095,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #endif
 #ifdef CONFIG_NEON32
 	SPEED_START {
-		raid_gen_force(6, raid_gen6_neon32_raid);
-		raid_recX_neon32(6, id, ip, nd, size, v);
+		raid_rec6_neon32(6, id, ip, nd, size, v);
 	} SPEED_STOP
 
 	printf("%8" PRIu64, ds / dt);
@@ -1812,36 +2105,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 #ifdef CONFIG_X86
 	if (raid_cpu_has_ssse3()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(6, raid_gen6_ssse3ext_raid);
-#else
-			raid_gen_force(6, raid_gen6_ssse3_raid);
-#endif
-			raid_recX_ssse3(6, id, ip, nd, size, v);
+			raid_rec6_ssse3(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
+
+#ifdef CONFIG_X86_64
+		SPEED_START {
+			raid_rec6_ssse3ext(6, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+#endif
 	}
 	if (raid_cpu_has_avx2()) {
 		SPEED_START {
-#ifdef CONFIG_X86_64
-			raid_gen_force(6, raid_gen6_avx2ext_raid);
-#else
-			raid_gen_force(6, raid_gen6_ssse3_raid);
-#endif
-			raid_recX_avx2(6, id, ip, nd, size, v);
+			raid_rec6_avx2(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
 		fflush(stdout);
 	}
 #ifdef CONFIG_X86_64
+	if (raid_cpu_has_avx2()) {
+		SPEED_START {
+			raid_rec6_avx2ext(6, id, ip, nd, size, v);
+		} SPEED_STOP
+
+		printf("%8" PRIu64, ds / dt);
+		fflush(stdout);
+	}
 	if (raid_cpu_has_avx512bw()) {
 		SPEED_START {
-			raid_gen_force(6, raid_gen6_avx512bw);
-			/* +1 to avoid GEN1 optimized case */
-			raid_recX_avx512bw(6, id, ip, nd, size, v);
+			raid_rec6_avx512bw(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1849,8 +2147,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx2gfni()) {
 		SPEED_START {
-			raid_gen_force(6, raid_gen6_avx2gfni_raid);
-			raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
+			raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);
@@ -1858,8 +2155,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
 	}
 	if (raid_cpu_has_avx512gfni()) {
 		SPEED_START {
-			raid_gen_force(6, raid_gen6_avx512gfni_raid);
-			raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
+			raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
 		} SPEED_STOP
 
 		printf("%8" PRIu64, ds / dt);