src/plugins/dpdk/cryptodev/cryptodev.h

   1 /*
   2  *------------------------------------------------------------------
   3  * Copyright (c) 2019 - 2021 Intel and/or its affiliates.
   4  * Licensed under the Apache License, Version 2.0 (the "License");
   5  * you may not use this file except in compliance with the License.
   6  * You may obtain a copy of the License at:
   7  *
   8  *     http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS,
  12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13  * See the License for the specific language governing permissions and
  14  * limitations under the License.
  15  *------------------------------------------------------------------
  16  */
  17 #ifndef included_cryptodev_h
  18 #define included_cryptodev_h
  19
  20 #include <vnet/crypto/crypto.h>
  21 #undef always_inline
  22 #include <rte_cryptodev.h>
  23
  24 #define CRYPTODEV_NB_CRYPTO_OPS    1024
  25 #define CRYPTODEV_CACHE_QUEUE_SIZE VNET_CRYPTO_FRAME_POOL_SIZE
  26 #define CRYPTODEV_CACHE_QUEUE_MASK (VNET_CRYPTO_FRAME_POOL_SIZE - 1)
  27 #define CRYPTODEV_MAX_INFLIGHT     (CRYPTODEV_NB_CRYPTO_OPS - 1)
  28 #define CRYPTODEV_AAD_MASK         (CRYPTODEV_NB_CRYPTO_OPS - 1)
  29 #define CRYPTODE_ENQ_MAX           64
  30 #define CRYPTODE_DEQ_MAX           64
  31 #define CRYPTODEV_NB_SESSION       4096
  32 #define CRYPTODEV_MAX_IV_SIZE      16
  33 #define CRYPTODEV_MAX_AAD_SIZE     16
  34 #define CRYPTODEV_MAX_N_SGL        8 /**< maximum number of segments */
  35
  36 #define CRYPTODEV_IV_OFFSET  (offsetof (cryptodev_op_t, iv))
  37 #define CRYPTODEV_AAD_OFFSET (offsetof (cryptodev_op_t, aad))
  38
  39 /* VNET_CRYPTO_ALGO, TYPE, DPDK_CRYPTO_ALGO, IV_LEN, TAG_LEN, AAD_LEN, KEY_LEN
  40  */
  41 #define foreach_vnet_aead_crypto_conversion                                   \
  42   _ (AES_128_GCM, AEAD, AES_GCM, 12, 16, 8, 16)                               \
  43   _ (AES_128_GCM, AEAD, AES_GCM, 12, 16, 12, 16)                              \
  44   _ (AES_192_GCM, AEAD, AES_GCM, 12, 16, 8, 24)                               \
  45   _ (AES_192_GCM, AEAD, AES_GCM, 12, 16, 12, 24)                              \
  46   _ (AES_256_GCM, AEAD, AES_GCM, 12, 16, 8, 32)                               \
  47   _ (AES_256_GCM, AEAD, AES_GCM, 12, 16, 12, 32)                              \
  48   _ (CHACHA20_POLY1305, AEAD, CHACHA20_POLY1305, 12, 16, 0, 32)               \
  49   _ (CHACHA20_POLY1305, AEAD, CHACHA20_POLY1305, 12, 16, 8, 32)               \
  50   _ (CHACHA20_POLY1305, AEAD, CHACHA20_POLY1305, 12, 16, 12, 32)
  51
  52 /**
  53  * crypto (alg, cryptodev_alg, key_size), hash (alg, digest-size)
  54  **/
  55 #define foreach_cryptodev_link_async_alg                                      \
  56   _ (AES_128_CBC, AES_CBC, 16, MD5, 12)                                       \
  57   _ (AES_192_CBC, AES_CBC, 24, MD5, 12)                                       \
  58   _ (AES_256_CBC, AES_CBC, 32, MD5, 12)                                       \
  59   _ (AES_128_CBC, AES_CBC, 16, SHA1, 12)                                      \
  60   _ (AES_192_CBC, AES_CBC, 24, SHA1, 12)                                      \
  61   _ (AES_256_CBC, AES_CBC, 32, SHA1, 12)                                      \
  62   _ (AES_128_CBC, AES_CBC, 16, SHA224, 14)                                    \
  63   _ (AES_192_CBC, AES_CBC, 24, SHA224, 14)                                    \
  64   _ (AES_256_CBC, AES_CBC, 32, SHA224, 14)                                    \
  65   _ (AES_128_CBC, AES_CBC, 16, SHA256, 16)                                    \
  66   _ (AES_192_CBC, AES_CBC, 24, SHA256, 16)                                    \
  67   _ (AES_256_CBC, AES_CBC, 32, SHA256, 16)                                    \
  68   _ (AES_128_CBC, AES_CBC, 16, SHA384, 24)                                    \
  69   _ (AES_192_CBC, AES_CBC, 24, SHA384, 24)                                    \
  70   _ (AES_256_CBC, AES_CBC, 32, SHA384, 24)                                    \
  71   _ (AES_128_CBC, AES_CBC, 16, SHA512, 32)                                    \
  72   _ (AES_192_CBC, AES_CBC, 24, SHA512, 32)                                    \
  73   _ (AES_256_CBC, AES_CBC, 32, SHA512, 32)                                    \
  74   _ (AES_128_CTR, AES_CTR, 16, SHA1, 12)                                      \
  75   _ (AES_192_CTR, AES_CTR, 24, SHA1, 12)                                      \
  76   _ (AES_256_CTR, AES_CTR, 32, SHA1, 12)
  77
  78 typedef enum
  79 {
  80   CRYPTODEV_OP_TYPE_ENCRYPT = 0,
  81   CRYPTODEV_OP_TYPE_DECRYPT,
  82   CRYPTODEV_N_OP_TYPES,
  83 } cryptodev_op_type_t;
  84
  85 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)
  86 typedef void cryptodev_session_t;
  87 #else
  88 typedef struct rte_cryptodev_sym_session cryptodev_session_t;
  89 #endif
  90
  91 /* Cryptodev session data, one data per direction per numa */
  92 typedef struct
  93 {
  94   cryptodev_session_t ***keys;
  95 } cryptodev_key_t;
  96
  97 /* Replicate DPDK rte_cryptodev_sym_capability structure with key size ranges
  98  * in favor of vpp vector */
  99 typedef struct
 100 {
 101   enum rte_crypto_sym_xform_type xform_type;
 102   union
 103   {
 104     struct
 105     {
 106       enum rte_crypto_auth_algorithm algo; /*auth algo */
 107       u32 *digest_sizes;                   /* vector of auth digest sizes */
 108     } auth;
 109     struct
 110     {
 111       enum rte_crypto_cipher_algorithm algo; /* cipher algo */
 112       u32 *key_sizes;                        /* vector of cipher key sizes */
 113     } cipher;
 114     struct
 115     {
 116       enum rte_crypto_aead_algorithm algo; /* aead algo */
 117       u32 *key_sizes;                      /*vector of aead key sizes */
 118       u32 *aad_sizes;                      /*vector of aad sizes */
 119       u32 *digest_sizes;                   /* vector of aead digest sizes */
 120     } aead;
 121   };
 122 } cryptodev_capability_t;
 123
 124 /* Cryptodev instance data */
 125 typedef struct
 126 {
 127   u32 dev_id;
 128   u32 q_id;
 129   char *desc;
 130 } cryptodev_inst_t;
 131
 132 typedef struct
 133 {
 134   struct rte_mempool *sess_pool;
 135 #if RTE_VERSION < RTE_VERSION_NUM(22, 11, 0, 0)
 136   struct rte_mempool *sess_priv_pool;
 137 #endif
 138 } cryptodev_session_pool_t;
 139
 140 typedef struct
 141 {
 142   cryptodev_session_pool_t *sess_pools;
 143 } cryptodev_numa_data_t;
 144
 145 typedef struct
 146 {
 147   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 148   struct rte_crypto_op op;
 149   struct rte_crypto_sym_op sop;
 150   u8 iv[CRYPTODEV_MAX_IV_SIZE];
 151   u8 aad[CRYPTODEV_MAX_AAD_SIZE];
 152   vnet_crypto_async_frame_t *frame;
 153   u32 n_elts;
 154 } cryptodev_op_t;
 155
 156 typedef struct
 157 {
 158   vnet_crypto_async_frame_t *f;
 159
 160   u8 enqueued;
 161   u8 dequeued;
 162   u8 deq_state;
 163   u8 frame_inflight;
 164
 165   u8 op_type;
 166   u8 aad_len;
 167   u8 n_elts;
 168   u8 reserved;
 169 } cryptodev_async_ring_elt;
 170
 171 typedef struct
 172 {
 173   cryptodev_async_ring_elt frames[VNET_CRYPTO_FRAME_POOL_SIZE];
 174   uint16_t head;
 175   uint16_t tail;
 176   uint16_t enq; /*record the frame currently being enqueued */
 177   uint16_t deq; /*record the frame currently being dequeued */
 178 } cryptodev_async_frame_sw_ring;
 179
 180 typedef struct
 181 {
 182   CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
 183   vlib_buffer_t *b[VNET_CRYPTO_FRAME_SIZE];
 184   union
 185   {
 186     struct
 187     {
 188       cryptodev_op_t **cops;
 189       struct rte_mempool *cop_pool;
 190       struct rte_ring *ring;
 191     };
 192     struct
 193     {
 194       struct rte_crypto_raw_dp_ctx *ctx;
 195       u16 aad_index;
 196       u8 *aad_buf;
 197       u64 aad_phy_addr;
 198       cryptodev_session_t *reset_sess;
 199     };
 200   };
 201
 202   cryptodev_async_frame_sw_ring frame_ring;
 203   u16 cryptodev_id;
 204   u16 cryptodev_q;
 205   u16 frames_on_ring;
 206   u16 enqueued_not_dequeueq;
 207   u16 deqeued_not_returned;
 208   u16 pending_to_qat;
 209   u16 inflight;
 210 } cryptodev_engine_thread_t;
 211
 212 typedef struct
 213 {
 214   cryptodev_numa_data_t *per_numa_data;
 215   cryptodev_key_t *keys;
 216   cryptodev_engine_thread_t *per_thread_data;
 217   enum rte_iova_mode iova_mode;
 218   cryptodev_inst_t *cryptodev_inst;
 219   clib_bitmap_t *active_cdev_inst_mask;
 220   clib_spinlock_t tlock;
 221   cryptodev_capability_t *supported_caps;
 222   u32 sess_sz;
 223   u32 drivers_cnt;
 224   u8 is_raw_api;
 225 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)
 226   u8 driver_id;
 227 #endif
 228 } cryptodev_main_t;
 229
 230 extern cryptodev_main_t cryptodev_main;
 231
 232 static_always_inline void
 233 cryptodev_mark_frame_err_status (vnet_crypto_async_frame_t *f,
 234                                  vnet_crypto_op_status_t s,
 235                                  vnet_crypto_async_frame_state_t fs)
 236 {
 237   u32 n_elts = f->n_elts, i;
 238
 239   for (i = 0; i < n_elts; i++)
 240     f->elts[i].status = s;
 241   f->state = fs;
 242 }
 243
 244 int cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
 245                               u32 aad_len);
 246
 247 void cryptodev_sess_handler (vlib_main_t *vm, vnet_crypto_key_op_t kop,
 248                              vnet_crypto_key_index_t idx, u32 aad_len);
 249
 250 int cryptodev_check_cap_support (struct rte_cryptodev_sym_capability_idx *idx,
 251                                  u32 key_size, u32 digest_size, u32 aad_size);
 252
 253 clib_error_t *cryptodev_register_cop_hdl (vlib_main_t *vm, u32 eidx);
 254
 255 clib_error_t *__clib_weak cryptodev_register_raw_hdl (vlib_main_t *vm,
 256                                                       u32 eidx);
 257
 258 clib_error_t *__clib_weak dpdk_cryptodev_init (vlib_main_t *vm);
 259
 260 #endif