src/plugins/crypto_native/aes_ctr.c

   1 /* SPDX-License-Identifier: Apache-2.0
   2  * Copyright(c) 2024 Cisco Systems, Inc.
   3  */
   4
   5 #include <vlib/vlib.h>
   6 #include <vnet/plugin/plugin.h>
   7 #include <vnet/crypto/crypto.h>
   8 #include <crypto_native/crypto_native.h>
   9 #include <vppinfra/crypto/aes_ctr.h>
  10
  11 #if __GNUC__ > 4 && !__clang__ && CLIB_DEBUG == 0
  12 #pragma GCC optimize("O3")
  13 #endif
  14
  15 static_always_inline u32
  16 aes_ops_aes_ctr (vlib_main_t *vm, vnet_crypto_op_t *ops[], u32 n_ops,
  17                  vnet_crypto_op_chunk_t *chunks, aes_key_size_t ks,
  18                  int maybe_chained)
  19 {
  20   crypto_native_main_t *cm = &crypto_native_main;
  21   vnet_crypto_op_t *op = ops[0];
  22   aes_ctr_key_data_t *kd;
  23   aes_ctr_ctx_t ctx;
  24   u32 n_left = n_ops;
  25
  26 next:
  27   kd = (aes_ctr_key_data_t *) cm->key_data[op->key_index];
  28
  29   clib_aes_ctr_init (&ctx, kd, op->iv, ks);
  30   if (op->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS)
  31     {
  32       vnet_crypto_op_chunk_t *chp = chunks + op->chunk_index;
  33       for (int j = 0; j < op->n_chunks; j++, chp++)
  34         clib_aes_ctr_transform (&ctx, chp->src, chp->dst, chp->len, ks);
  35     }
  36   else
  37     clib_aes_ctr_transform (&ctx, op->src, op->dst, op->len, ks);
  38
  39   op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
  40
  41   if (--n_left)
  42     {
  43       op += 1;
  44       goto next;
  45     }
  46
  47   return n_ops;
  48 }
  49
  50 static_always_inline void *
  51 aes_ctr_key_exp (vnet_crypto_key_t *key, aes_key_size_t ks)
  52 {
  53   aes_ctr_key_data_t *kd;
  54
  55   kd = clib_mem_alloc_aligned (sizeof (*kd), CLIB_CACHE_LINE_BYTES);
  56
  57   clib_aes_ctr_key_expand (kd, key->data, ks);
  58
  59   return kd;
  60 }
  61
  62 #define foreach_aes_ctr_handler_type _ (128) _ (192) _ (256)
  63
  64 #define _(x)                                                                  \
  65   static u32 aes_ops_aes_ctr_##x (vlib_main_t *vm, vnet_crypto_op_t *ops[],   \
  66                                   u32 n_ops)                                  \
  67   {                                                                           \
  68     return aes_ops_aes_ctr (vm, ops, n_ops, 0, AES_KEY_##x, 0);               \
  69   }                                                                           \
  70   static u32 aes_ops_aes_ctr_##x##_chained (                                  \
  71     vlib_main_t *vm, vnet_crypto_op_t *ops[], vnet_crypto_op_chunk_t *chunks, \
  72     u32 n_ops)                                                                \
  73   {                                                                           \
  74     return aes_ops_aes_ctr (vm, ops, n_ops, chunks, AES_KEY_##x, 1);          \
  75   }                                                                           \
  76   static void *aes_ctr_key_exp_##x (vnet_crypto_key_t *key)                   \
  77   {                                                                           \
  78     return aes_ctr_key_exp (key, AES_KEY_##x);                                \
  79   }
  80
  81 foreach_aes_ctr_handler_type;
  82 #undef _
  83
  84 static int
  85 probe ()
  86 {
  87 #if defined(__VAES__) && defined(__AVX512F__)
  88   if (clib_cpu_supports_vaes () && clib_cpu_supports_avx512f ())
  89     return 50;
  90 #elif defined(__VAES__)
  91   if (clib_cpu_supports_vaes ())
  92     return 40;
  93 #elif defined(__AVX512F__)
  94   if (clib_cpu_supports_avx512f ())
  95     return 30;
  96 #elif defined(__AVX2__)
  97   if (clib_cpu_supports_avx2 ())
  98     return 20;
  99 #elif __AES__
 100   if (clib_cpu_supports_aes ())
 101     return 10;
 102 #elif __aarch64__
 103   if (clib_cpu_supports_aarch64_aes ())
 104     return 10;
 105 #endif
 106   return -1;
 107 }
 108
 109 #define _(b)                                                                  \
 110   CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_enc) = {                            \
 111     .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_ENC,                                \
 112     .fn = aes_ops_aes_ctr_##b,                                                \
 113     .cfn = aes_ops_aes_ctr_##b##_chained,                                     \
 114     .probe = probe,                                                           \
 115   };                                                                          \
 116                                                                               \
 117   CRYPTO_NATIVE_OP_HANDLER (aes_##b##_ctr_dec) = {                            \
 118     .op_id = VNET_CRYPTO_OP_AES_##b##_CTR_DEC,                                \
 119     .fn = aes_ops_aes_ctr_##b,                                                \
 120     .cfn = aes_ops_aes_ctr_##b##_chained,                                     \
 121     .probe = probe,                                                           \
 122   };                                                                          \
 123   CRYPTO_NATIVE_KEY_HANDLER (aes_##b##_ctr) = {                               \
 124     .alg_id = VNET_CRYPTO_ALG_AES_##b##_CTR,                                  \
 125     .key_fn = aes_ctr_key_exp_##b,                                            \
 126     .probe = probe,                                                           \
 127   };
 128
 129 _ (128) _ (192) _ (256)
 130 #undef _