Split GEN x2 and G23 paths to eliminate AES hot-path overhead

raid/internal.h

@@ -75,7 +75,15 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
 }
 
 /*
- * Initialize the countdown used by the AES G23 Horner recurrence.
+ * Check if the number of data disks requires only x2 multipliers in AES G23.
+ */
+static __always_inline int raid_g23_x2_only(int nd)
+{
+	return nd <= 51;
+}
+
+/*
+ * Find the next AES G23 x3 boundary in descending Horner order.
  *
  * The G23 Q coefficients are generated by multiplying by 2 at every step,
  * except at the four boundaries between the five cosets, where the next
@@ -92,43 +100,53 @@ static __always_inline void * __align_ptr(void *ptr, uintptr_t size)
  *
  * i.e. every 51 descending steps.
  *
- * Rather than comparing d against those four values in the hot loop, we keep
- * a countdown. Before processing each Horner step the generator does:
+ * The generators can process all the x2 transitions between two consecutive
+ * boundaries in a branch-free loop, and handle the x3 transition separately.
+ * This function returns the first x3 boundary encountered by a descending
+ * traversal whose first processed step has index d.
  *
- *     if (--g23_count == 0) {
- *         Q = 3 * Q;
- *         g23_count = 51;
- *     } else {
- *         Q = 2 * Q;
- *     }
+ * For d >= 50, (d - 50) % 51 gives the distance from the previous x3
+ * boundary in ascending index order. Subtracting this distance from d gives
+ * that boundary directly:
  *
- * This function returns the initial countdown for a traversal whose first
- * processed step has index d. For d >= 50, (d - 50) % 51 gives the distance
- * from the previous x3 boundary, so adding 1 gives the number of decrements
- * required before the next descending x3 transition is reached.
+ *     boundary = d - (d - 50) % 51
  *
- * For d < 50 there is no remaining x3 transition. Returning 51 is safe
- * because fewer than 51 Horner steps remain, so the countdown cannot reach
- * zero before the traversal ends.
+ * Since the Horner traversal proceeds toward decreasing indexes, this is also
+ * the next x3 transition that will be encountered.
+ *
+ * For d < 50 there is no remaining x3 transition, so -1 is returned. This
+ * value can also be used as the lower bound of the final x2-only segment,
+ * allowing all remaining steps down to d = 0 to be processed without a
+ * special G23 test.
  *
  * Examples:
  *
- *     d = 203 ->  1  -> x3 immediately
- *     d = 204 ->  2  -> x2 at 204, x3 at 203
- *     d = 250 -> 48  -> x3 at 203
- *     d = 152 ->  1  -> x3 immediately
- *     d =  50 ->  1  -> x3 immediately
- *     d =  49 -> 51  -> no further x3 transition
+ *     d = 249 -> 203 -> x2 down to 204, then x3 at 203
+ *     d = 203 -> 203 -> x3 immediately
+ *     d = 202 -> 152 -> x2 down to 153, then x3 at 152
+ *     d = 152 -> 152 -> x3 immediately
+ *     d = 151 -> 101 -> x2 down to 102, then x3 at 101
+ *     d = 101 -> 101 -> x3 immediately
+ *     d = 100 ->  50 -> x2 down to 51, then x3 at 50
+ *     d =  50 ->  50 -> x3 immediately
+ *     d =  49 ->  -1 -> no further x3 transition
  *
- * The returned value is normally computed once as g23_start and copied into
- * g23_count at the beginning of each independent Horner traversal.
+ * After processing a boundary, the following one in descending order is
+ * exactly 51 positions lower. Thus generators can continue with:
+ *
+ *     boundary -= 51;
+ *     if (boundary < 50)
+ *         boundary = -1;
+ *
+ * The returned value is normally computed once before each independent
+ * Horner traversal and then advanced between the fixed G23 segments.
  */
-static __always_inline int raid_g23_count(int d)
+static __always_inline int raid_g23_boundary(int d)
 {
 	if (d < 50)
-		return 51;
+		return -1;
 
-	return (d - 50) % 51 + 1;
+	return d - (d - 50) % 51;
 }
 
 /*
@@ -152,7 +170,6 @@ void raid_gen1_int64(int nd, size_t size, void **vv);
 void raid_gen1_sse2(int nd, size_t size, void **vv);
 void raid_gen1_avx2(int nd, size_t size, void **vv);
 void raid_gen1_avx512bw(int nd, size_t size, void **vv);
-void raid_gen2_int8(int nd, size_t size, void **vv);
 void raid_gen2_int32_raid(int nd, size_t size, void **vv);
 void raid_gen2_int32_aes(int nd, size_t size, void **vv);
 void raid_gen2_int64_raid(int nd, size_t size, void **vv);
@@ -201,6 +218,7 @@ void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv);
 void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv);
 void raid_gen5_int8(int nd, size_t size, void **vv);
 void raid_gen5_ssse3_raid(int nd, size_t size, void **vv);
+void raid_gen5_ssse3_aes(int nd, size_t size, void **vv);
 void raid_gen5_ssse3ext_raid(int nd, size_t size, void **vv);
 void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv);
 void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv);
@@ -212,6 +230,7 @@ void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv);
 void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv);
 void raid_gen6_int8(int nd, size_t size, void **vv);
 void raid_gen6_ssse3_raid(int nd, size_t size, void **vv);
+void raid_gen6_ssse3_aes(int nd, size_t size, void **vv);
 void raid_gen6_ssse3ext_raid(int nd, size_t size, void **vv);
 void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv);
 void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv);