X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fcrypto_ia32%2Faes_cbc.c;h=4ad01b52b29ade4bc15137d3ca778d3be6e17ea2;hb=4ed2598562b1fe65efe69ae53397dfe9f1f07be8;hp=281cc83705a6f575d420a62016507d0843cb8d38;hpb=deb8af6eb71f9ad951137ee6e84e971c3ab23ec4;p=vpp.git

diff --git a/src/plugins/crypto_ia32/aes_cbc.c b/src/plugins/crypto_ia32/aes_cbc.c
index 281cc83705a..4ad01b52b29 100644
--- a/src/plugins/crypto_ia32/aes_cbc.c
+++ b/src/plugins/crypto_ia32/aes_cbc.c
@@ -22,6 +22,16 @@
 #include <crypto_ia32/crypto_ia32.h>
 #include <crypto_ia32/aesni.h>
 
+#if __GNUC__ > 4  && !__clang__ && CLIB_DEBUG == 0
+#pragma GCC optimize ("O3")
+#endif
+
+typedef struct
+{
+  __m128i encrypt_key[15];
+  __m128i decrypt_key[15];
+} aes_cbc_key_data_t;
+
 static_always_inline void
 aes_cbc_dec (__m128i * k, u8 * src, u8 * dst, u8 * iv, int count,
 	     aesni_key_size_t rounds)
@@ -95,10 +105,13 @@ aesni_ops_enc_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
 							 vm->thread_index);
   int rounds = AESNI_KEY_ROUNDS (ks);
   u8 dummy[8192];
-  u8 *src[4], *dst[4], *key[4];
-  u32x4 dummy_mask, len = { };
+  u8 *src[4] = { };
+  u8 *dst[4] = { };
+  vnet_crypto_key_index_t key_index[4] = { ~0, ~0, ~0, ~0 };
+  u32x4 dummy_mask = { };
+  u32x4 len = { };
   u32 i, j, count, n_left = n_ops;
-  __m128i r[4], k[4][rounds + 1];
+  __m128i r[4] = { }, k[4][rounds + 1];
 
 more:
   for (i = 0; i < 4; i++)
@@ -125,10 +138,13 @@ more:
 	    dst[i] = ops[0]->dst;
 	    len[i] = ops[0]->len;
 	    dummy_mask[i] = ~0;
-	    if (key[i] != ops[0]->key)
+	    if (key_index[i] != ops[0]->key_index)
 	      {
-		aes_key_expand (k[i], ops[0]->key, ks);
-		key[i] = ops[0]->key;
+		aes_cbc_key_data_t *kd;
+		key_index[i] = ops[0]->key_index;
+		kd = (aes_cbc_key_data_t *) cm->key_data[key_index[i]];
+		clib_memcpy_fast (k[i], kd->encrypt_key,
+				  (rounds + 1) * sizeof (__m128i));
 	      }
 	    ops[0]->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
 	    n_left--;
@@ -186,26 +202,39 @@ static_always_inline u32
 aesni_ops_dec_aes_cbc (vlib_main_t * vm, vnet_crypto_op_t * ops[],
 		       u32 n_ops, aesni_key_size_t ks)
 {
+  crypto_ia32_main_t *cm = &crypto_ia32_main;
   int rounds = AESNI_KEY_ROUNDS (ks);
-  u8 *last_key = 0;
-  u32 i;
-  __m128i k[rounds + 1];
+  vnet_crypto_op_t *op = ops[0];
+  aes_cbc_key_data_t *kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
+  u32 n_left = n_ops;
+
+  ASSERT (n_ops >= 1);
+
+decrypt:
+  aes_cbc_dec (kd->decrypt_key, op->src, op->dst, op->iv, op->len, rounds);
+  op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
 
-  for (i = 0; i < n_ops; i++)
+  if (--n_left)
     {
-      vnet_crypto_op_t *op = ops[i];
-      if (last_key != op->key)
-	{
-	  aes_key_expand (k, op->key, ks);
-	  last_key = op->key;
-	  aes_key_enc_to_dec (k, rounds);
-	}
-      aes_cbc_dec (k, op->src, op->dst, op->iv, op->len, rounds);
-      op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
+      op += 1;
+      kd = (aes_cbc_key_data_t *) cm->key_data[op->key_index];
+      goto decrypt;
     }
+
   return n_ops;
 }
 
+static_always_inline void *
+aesni_cbc_key_exp (vnet_crypto_key_t * key, aesni_key_size_t ks)
+{
+  aes_cbc_key_data_t *kd;
+  kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
+  aes_key_expand (kd->encrypt_key, key->data, ks);
+  aes_key_expand (kd->decrypt_key, key->data, ks);
+  aes_key_enc_to_dec (kd->decrypt_key, ks);
+  return kd;
+}
+
 #define foreach_aesni_cbc_handler_type _(128) _(192) _(256)
 
 #define _(x) \
@@ -215,6 +244,8 @@ static u32 aesni_ops_dec_aes_cbc_##x \
 static u32 aesni_ops_enc_aes_cbc_##x \
 (vlib_main_t * vm, vnet_crypto_op_t * ops[], u32 n_ops) \
 { return aesni_ops_enc_aes_cbc (vm, ops, n_ops, AESNI_KEY_##x); } \
+static void * aesni_cbc_key_exp_##x (vnet_crypto_key_t *key) \
+{ return aesni_cbc_key_exp (key, AESNI_KEY_##x); }
 
 foreach_aesni_cbc_handler_type;
 #undef _
@@ -222,7 +253,13 @@ foreach_aesni_cbc_handler_type;
 #include <fcntl.h>
 
 clib_error_t *
-crypto_ia32_aesni_cbc_init (vlib_main_t * vm)
+#ifdef __AVX512F__
+crypto_ia32_aesni_cbc_init_avx512 (vlib_main_t * vm)
+#elif __AVX2__
+crypto_ia32_aesni_cbc_init_avx2 (vlib_main_t * vm)
+#else
+crypto_ia32_aesni_cbc_init_sse42 (vlib_main_t * vm)
+#endif
 {
   crypto_ia32_main_t *cm = &crypto_ia32_main;
   crypto_ia32_per_thread_data_t *ptd;
@@ -237,7 +274,8 @@ crypto_ia32_aesni_cbc_init (vlib_main_t * vm)
     {
       for (int i = 0; i < 4; i++)
 	{
-	  if (read(fd, ptd->cbc_iv, sizeof (ptd->cbc_iv)) < 0)
+	  if (read(fd, ptd->cbc_iv, sizeof (ptd->cbc_iv)) !=
+	      sizeof (ptd->cbc_iv))
 	    {
 	      err = clib_error_return_unix (0, "'/dev/urandom' read failure");
 	      goto error;
@@ -252,7 +290,8 @@ crypto_ia32_aesni_cbc_init (vlib_main_t * vm)
 				    aesni_ops_enc_aes_cbc_##x); \
   vnet_crypto_register_ops_handler (vm, cm->crypto_engine_index, \
 				    VNET_CRYPTO_OP_AES_##x##_CBC_DEC, \
-				    aesni_ops_dec_aes_cbc_##x);
+				    aesni_ops_dec_aes_cbc_##x); \
+  cm->key_fn[VNET_CRYPTO_ALG_AES_##x##_CBC] = aesni_cbc_key_exp_##x;
   foreach_aesni_cbc_handler_type;
 #undef _