vppinfra: add 128-bit and 512-bit a ^ b ^ c shortcut

[vpp.git] / src / plugins / crypto_native / ghash.h

diff --git a/src/plugins/crypto_native/ghash.h b/src/plugins/crypto_native/ghash.h
index 3f68f80..a2886a4 100644 (file)
--- a/src/plugins/crypto_native/ghash.h
+++ b/src/plugins/crypto_native/ghash.h
@@ -105,40 +105,47 @@
 #ifndef __ghash_h__
 #define __ghash_h__
 
-/* on AVX-512 systems we can save a clock cycle by using ternary logic
-   instruction to calculate a XOR b XOR c */
-static_always_inline u8x16
-ghash_xor3 (u8x16 a, u8x16 b, u8x16 c)
-{
-#if defined (__AVX512F__)
-  return (u8x16) _mm_ternarylogic_epi32 ((__m128i) a, (__m128i) b,
-                                        (__m128i) c, 0x96);
-#endif
-  return a ^ b ^ c;
-}
-
 static_always_inline u8x16
 gmul_lo_lo (u8x16 a, u8x16 b)
 {
+#if defined (__PCLMUL__)
   return (u8x16) _mm_clmulepi64_si128 ((__m128i) a, (__m128i) b, 0x00);
+#elif defined (__ARM_FEATURE_CRYPTO)
+  return (u8x16) vmull_p64 ((poly64_t) vget_low_p64 ((poly64x2_t) a),
+                           (poly64_t) vget_low_p64 ((poly64x2_t) b));
+#endif
 }
 
 static_always_inline u8x16
-gmul_lo_hi (u8x16 a, u8x16 b)
+gmul_hi_lo (u8x16 a, u8x16 b)
 {
+#if defined (__PCLMUL__)
   return (u8x16) _mm_clmulepi64_si128 ((__m128i) a, (__m128i) b, 0x01);
+#elif defined (__ARM_FEATURE_CRYPTO)
+  return (u8x16) vmull_p64 ((poly64_t) vget_high_p64 ((poly64x2_t) a),
+                           (poly64_t) vget_low_p64 ((poly64x2_t) b));
+#endif
 }
 
 static_always_inline u8x16
-gmul_hi_lo (u8x16 a, u8x16 b)
+gmul_lo_hi (u8x16 a, u8x16 b)
 {
+#if defined (__PCLMUL__)
   return (u8x16) _mm_clmulepi64_si128 ((__m128i) a, (__m128i) b, 0x10);
+#elif defined (__ARM_FEATURE_CRYPTO)
+  return (u8x16) vmull_p64 ((poly64_t) vget_low_p64 ((poly64x2_t) a),
+                           (poly64_t) vget_high_p64 ((poly64x2_t) b));
+#endif
 }
 
 static_always_inline u8x16
 gmul_hi_hi (u8x16 a, u8x16 b)
 {
+#if defined (__PCLMUL__)
   return (u8x16) _mm_clmulepi64_si128 ((__m128i) a, (__m128i) b, 0x11);
+#elif defined (__ARM_FEATURE_CRYPTO)
+  return (u8x16) vmull_high_p64 ((poly64x2_t) a, (poly64x2_t) b);
+#endif
 }
 
 typedef struct
@@ -165,7 +172,7 @@ ghash_mul_first (ghash_data_t * gd, u8x16 a, u8x16 b)
   /* a0 * b0 */
   gd->lo = gmul_lo_lo (a, b);
   /* a0 * b1 ^ a1 * b0 */
-  gd->mid = (gmul_lo_hi (a, b) ^ gmul_hi_lo (a, b));
+  gd->mid = (gmul_hi_lo (a, b) ^ gmul_lo_hi (a, b));
 
   /* set gd->pending to 0 so next invocation of ghash_mul_next(...) knows that
      there is no pending data in tmp_lo and tmp_hi */
@@ -185,8 +192,8 @@ ghash_mul_next (ghash_data_t * gd, u8x16 a, u8x16 b)
   if (gd->pending)
     {
       /* there is peding data from previous invocation so we can XOR */
-      gd->hi = ghash_xor3 (gd->hi, gd->tmp_hi, hi);
-      gd->lo = ghash_xor3 (gd->lo, gd->tmp_lo, lo);
+      gd->hi = u8x16_xor3 (gd->hi, gd->tmp_hi, hi);
+      gd->lo = u8x16_xor3 (gd->lo, gd->tmp_lo, lo);
       gd->pending = 0;
     }
   else
@@ -198,7 +205,7 @@ ghash_mul_next (ghash_data_t * gd, u8x16 a, u8x16 b)
     }
 
   /* gd->mid ^= a0 * b1 ^ a1 * b0  */
-  gd->mid = ghash_xor3 (gd->mid, gmul_lo_hi (a, b), gmul_hi_lo (a, b));
+  gd->mid = u8x16_xor3 (gd->mid, gmul_hi_lo (a, b), gmul_lo_hi (a, b));
 }
 
 static_always_inline void
@@ -214,16 +221,15 @@ ghash_reduce (ghash_data_t * gd)
 
   if (gd->pending)
     {
-      gd->lo = ghash_xor3 (gd->lo, gd->tmp_lo, midl);
-      gd->hi = ghash_xor3 (gd->hi, gd->tmp_hi, midr);
+      gd->lo = u8x16_xor3 (gd->lo, gd->tmp_lo, midl);
+      gd->hi = u8x16_xor3 (gd->hi, gd->tmp_hi, midr);
     }
   else
     {
       gd->lo ^= midl;
       gd->hi ^= midr;
     }
-
-  r = gmul_lo_hi (ghash_poly2, gd->lo);
+  r = gmul_hi_lo (ghash_poly2, gd->lo);
   gd->lo ^= u8x16_word_shift_left (r, 8);
 }
 
@@ -231,13 +237,13 @@ static_always_inline void
 ghash_reduce2 (ghash_data_t * gd)
 {
   gd->tmp_lo = gmul_lo_lo (ghash_poly2, gd->lo);
-  gd->tmp_hi = gmul_hi_lo (ghash_poly2, gd->lo);
+  gd->tmp_hi = gmul_lo_hi (ghash_poly2, gd->lo);
 }
 
 static_always_inline u8x16
 ghash_final (ghash_data_t * gd)
 {
-  return ghash_xor3 (gd->hi, u8x16_word_shift_right (gd->tmp_lo, 4),
+  return u8x16_xor3 (gd->hi, u8x16_word_shift_right (gd->tmp_lo, 4),
                     u8x16_word_shift_left (gd->tmp_hi, 4));
 }
 
@@ -261,7 +267,11 @@ ghash_precompute (u8x16 H, u8x16 * Hi, int count)
   H = (u8x16) ((u64x2) H << 1);
   H |= u8x16_word_shift_left (r8, 8);
   r32 = (u32x4) u8x16_word_shift_right (r8, 8);
+#ifdef __SSE2__
   r32 = u32x4_shuffle (r32, 0, 1, 2, 0);
+#else
+  r32[3] = r32[0];
+#endif
   /* *INDENT-OFF* */
   r32 = r32 == (u32x4) {1, 0, 0, 1};
   /* *INDENT-ON* */