src/plugins/crypto_native/aes_gcm.c

   1 /*
   2  *------------------------------------------------------------------
   3  * Copyright (c) 2019 Cisco and/or its affiliates.
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  *------------------------------------------------------------------
  16  */
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/plugin/plugin.h>
  20 #include <vnet/crypto/crypto.h>
  21 #include <crypto_native/crypto_native.h>
  22 #include <vppinfra/crypto/aes_gcm.h>
  23
  24 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
  25 #pragma GCC optimize("O3")
  26 #endif
  27
  28 static_always_inline u32
  29 aes_ops_enc_aes_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
  30                      aes_key_size_t ks)
  31 {
  32   crypto_native_main_t *cm = &crypto_native_main;
  33   vnet_crypto_op_t *op = ops[0];
  34   aes_gcm_key_data_t *kd;
  35   u32 n_left = n_ops;
  36
  37 next:
  38   kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
  39   aes_gcm (op->src, op->dst, op->aad, (u8 *) op->iv, op->tag, op->len,
  40            op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
  41            AES_GCM_OP_ENCRYPT);
  42   op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
  43
  44   if (--n_left)
  45     {
  46       op += 1;
  47       goto next;
  48     }
  49
  50   return n_ops;
  51 }
  52
  53 static_always_inline u32
  54 aes_ops_dec_aes_gcm (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
  55                      aes_key_size_t ks)
  56 {
  57   crypto_native_main_t *cm = &crypto_native_main;
  58   vnet_crypto_op_t *op = ops[0];
  59   aes_gcm_key_data_t *kd;
  60   u32 n_left = n_ops;
  61   int rv;
  62
  63 next:
  64   kd = (aes_gcm_key_data_t *) cm->key_data[op->key_index];
  65   rv = aes_gcm (op->src, op->dst, op->aad, (u8 *) op->iv, op->tag, op->len,
  66                 op->aad_len, op->tag_len, kd, AES_KEY_ROUNDS (ks),
  67                 AES_GCM_OP_DECRYPT);
  68
  69   if (rv)
  70     {
  71       op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
  72     }
  73   else
  74     {
  75       op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
  76       n_ops--;
  77     }
  78
  79   if (--n_left)
  80     {
  81       op += 1;
  82       goto next;
  83     }
  84
  85   return n_ops;
  86 }
  87
  88 static_always_inline void *
  89 aes_gcm_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks)
  90 {
  91   aes_gcm_key_data_t *kd;
  92
  93   kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
  94
  95   clib_aes_gcm_key_expand (kd, key->data, ks);
  96
  97   return kd;
  98 }
  99
 100 #define foreach_aes_gcm_handler_type _ (128) _ (192) _ (256)
 101
 102 #define _(x)                                                                  \
 103   static u32 aes_ops_dec_aes_gcm_##x (vlib_main_t *vm,                        \
 104                                       vnet_crypto_op_t *ops[], u32 n_ops)     \
 105   {                                                                           \
 106     return aes_ops_dec_aes_gcm (vm, ops, n_ops, AES_KEY_##x);                 \
 107   }                                                                           \
 108   static u32 aes_ops_enc_aes_gcm_##x (vlib_main_t *vm,                        \
 109                                       vnet_crypto_op_t *ops[], u32 n_ops)     \
 110   {                                                                           \
 111     return aes_ops_enc_aes_gcm (vm, ops, n_ops, AES_KEY_##x);                 \
 112   }                                                                           \
 113   static void *aes_gcm_key_exp_##x (vnet_crypto_key_t *key)                   \
 114   {                                                                           \
 115     return aes_gcm_key_exp (key, AES_KEY_##x);                                \
 116   }
 117
 118 foreach_aes_gcm_handler_type;
 119 #undef _
 120
 121 static int
 122 probe ()
 123 {
 124 #if defined(__VAES__) && defined(__AVX512F__)
 125   if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes () &&
 126       clib_cpu_supports_avx512f ())
 127     return 50;
 128 #elif defined(__VAES__)
 129   if (clib_cpu_supports_vpclmulqdq () && clib_cpu_supports_vaes ())
 130     return 40;
 131 #elif defined(__AVX512F__)
 132   if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx512f ())
 133     return 30;
 134 #elif defined(__AVX2__)
 135   if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_avx2 ())
 136     return 20;
 137 #elif __AES__
 138   if (clib_cpu_supports_pclmulqdq () && clib_cpu_supports_aes ())
 139     return 10;
 140 #elif __aarch64__
 141   if (clib_cpu_supports_aarch64_aes ())
 142     return 10;
 143 #endif
 144   return -1;
 145 }
 146
 147 #define _(b)                                                                  \
 148   CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_enc) = {                            \
 149     .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_ENC,                                \
 150     .fn = aes_ops_enc_aes_gcm_##b,                                            \
 151     .probe = probe,                                                           \
 152   };                                                                          \
 153                                                                               \
 154   CRYPTO_NATIVE_OP_HANDLER (aes_##b##_gcm_dec) = {                            \
 155     .op_id = VNET_CRYPTO_OP_AES_##b##_GCM_DEC,                                \
 156     .fn = aes_ops_dec_aes_gcm_##b,                                            \
 157     .probe = probe,                                                           \
 158   };                                                                          \
 159   CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_gcm) = {                               \
 160     .alg_id = VNET_CRYPTO_ALG_AES_##b##_GCM,                                  \
 161     .key_fn = aes_gcm_key_exp_##b,                                            \
 162     .probe = probe,                                                           \
 163   };
 164
 165 _ (128) _ (192) _ (256)
 166 #undef _