src/plugins/dpdk/ipsec/ipsec.c

   1 /*
   2  * Copyright (c) 2016 Intel and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <vnet/vnet.h>
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/api_errno.h>
  18 #include <vnet/ipsec/ipsec.h>
  19 #include <vlib/node_funcs.h>
  20
  21 #include <dpdk/device/dpdk.h>
  22 #include <dpdk/ipsec/ipsec.h>
  23 #include <dpdk/ipsec/esp.h>
  24
  25 #define DPDK_CRYPTO_NB_SESS_OBJS  20000
  26 #define DPDK_CRYPTO_CACHE_SIZE    512
  27 #define DPDK_CRYPTO_PRIV_SIZE     128
  28 #define DPDK_CRYPTO_N_QUEUE_DESC  1024
  29 #define DPDK_CRYPTO_NB_COPS       (1024 * 4)
  30
  31 static int
  32 add_del_sa_sess (u32 sa_index, u8 is_add)
  33 {
  34   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
  35   crypto_worker_main_t *cwm;
  36   u8 skip_master = vlib_num_workers () > 0;
  37
  38   /* *INDENT-OFF* */
  39   vec_foreach (cwm, dcm->workers_main)
  40     {
  41       crypto_sa_session_t *sa_sess;
  42       u8 is_outbound;
  43
  44       if (skip_master)
  45         {
  46           skip_master = 0;
  47           continue;
  48         }
  49
  50       for (is_outbound = 0; is_outbound < 2; is_outbound++)
  51         {
  52           if (is_add)
  53             {
  54               pool_get (cwm->sa_sess_d[is_outbound], sa_sess);
  55             }
  56           else
  57             {
  58               u8 dev_id;
  59               i32 ret;
  60
  61               sa_sess = pool_elt_at_index (cwm->sa_sess_d[is_outbound], sa_index);
  62               dev_id = cwm->qp_data[sa_sess->qp_index].dev_id;
  63
  64               if (!sa_sess->sess)
  65                 continue;
  66 #if DPDK_NO_AEAD
  67               ret = (rte_cryptodev_sym_session_free(dev_id, sa_sess->sess) == NULL);
  68               ASSERT (ret);
  69 #else
  70               ret = rte_cryptodev_sym_session_clear(dev_id, sa_sess->sess);
  71               ASSERT (!ret);
  72
  73               ret = rte_cryptodev_sym_session_free(sa_sess->sess);
  74               ASSERT (!ret);
  75 #endif
  76               memset(sa_sess, 0, sizeof(sa_sess[0]));
  77             }
  78         }
  79     }
  80   /* *INDENT-OFF* */
  81
  82   return 0;
  83 }
  84
  85 static void
  86 update_qp_data (crypto_worker_main_t * cwm,
  87                 u8 cdev_id, u16 qp_id, u8 is_outbound, u16 * idx)
  88 {
  89   crypto_qp_data_t *qpd;
  90
  91   /* *INDENT-OFF* */
  92   vec_foreach_index (*idx, cwm->qp_data)
  93     {
  94       qpd = vec_elt_at_index(cwm->qp_data, *idx);
  95
  96       if (qpd->dev_id == cdev_id && qpd->qp_id == qp_id &&
  97           qpd->is_outbound == is_outbound)
  98           return;
  99     }
 100   /* *INDENT-ON* */
 101
 102   vec_add2_aligned (cwm->qp_data, qpd, 1, CLIB_CACHE_LINE_BYTES);
 103
 104   qpd->dev_id = cdev_id;
 105   qpd->qp_id = qp_id;
 106   qpd->is_outbound = is_outbound;
 107 }
 108
 109 /*
 110  * return:
 111  *      0: already exist
 112  *      1: mapped
 113  */
 114 static int
 115 add_mapping (crypto_worker_main_t * cwm,
 116              u8 cdev_id, u16 qp, u8 is_outbound,
 117              const struct rte_cryptodev_capabilities *cipher_cap,
 118              const struct rte_cryptodev_capabilities *auth_cap)
 119 {
 120   u16 qp_index;
 121   uword key = 0, data, *ret;
 122   crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
 123
 124   p_key->cipher_algo = (u8) cipher_cap->sym.cipher.algo;
 125   p_key->auth_algo = (u8) auth_cap->sym.auth.algo;
 126   p_key->is_outbound = is_outbound;
 127 #if ! DPDK_NO_AEAD
 128   p_key->is_aead = cipher_cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD;
 129 #endif
 130
 131   ret = hash_get (cwm->algo_qp_map, key);
 132   if (ret)
 133     return 0;
 134
 135   update_qp_data (cwm, cdev_id, qp, is_outbound, &qp_index);
 136
 137   data = (uword) qp_index;
 138   hash_set (cwm->algo_qp_map, key, data);
 139
 140   return 1;
 141 }
 142
 143 /*
 144  * return:
 145  *      0: already exist
 146  *      1: mapped
 147  */
 148 static int
 149 add_cdev_mapping (crypto_worker_main_t * cwm,
 150                   struct rte_cryptodev_info *dev_info, u8 cdev_id,
 151                   u16 qp, u8 is_outbound)
 152 {
 153   const struct rte_cryptodev_capabilities *i, *j;
 154   u32 mapped = 0;
 155
 156   for (i = dev_info->capabilities; i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++)
 157     {
 158 #if ! DPDK_NO_AEAD
 159       if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD)
 160         {
 161           struct rte_cryptodev_capabilities none = { 0 };
 162
 163           if (check_algo_is_supported (i, NULL) != 0)
 164             continue;
 165
 166           none.sym.auth.algo = RTE_CRYPTO_AUTH_NULL;
 167
 168           mapped |= add_mapping (cwm, cdev_id, qp, is_outbound, i, &none);
 169           continue;
 170         }
 171 #endif
 172       if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
 173         continue;
 174
 175       if (check_algo_is_supported (i, NULL) != 0)
 176         continue;
 177
 178       for (j = dev_info->capabilities; j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED;
 179            j++)
 180         {
 181           if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
 182             continue;
 183
 184           if (check_algo_is_supported (j, NULL) != 0)
 185             continue;
 186
 187           mapped |= add_mapping (cwm, cdev_id, qp, is_outbound, i, j);
 188         }
 189     }
 190
 191   return mapped;
 192 }
 193
 194 static int
 195 check_cryptodev_queues ()
 196 {
 197   u32 n_qs = 0;
 198   u8 cdev_id;
 199   u32 n_req_qs = 2;
 200
 201   if (vlib_num_workers () > 0)
 202     n_req_qs = vlib_num_workers () * 2;
 203
 204   for (cdev_id = 0; cdev_id < rte_cryptodev_count (); cdev_id++)
 205     {
 206       struct rte_cryptodev_info cdev_info;
 207
 208       rte_cryptodev_info_get (cdev_id, &cdev_info);
 209
 210       if (!
 211           (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
 212         continue;
 213
 214       n_qs += cdev_info.max_nb_queue_pairs;
 215     }
 216
 217   if (n_qs >= n_req_qs)
 218     return 0;
 219   else
 220     return -1;
 221 }
 222
 223 static clib_error_t *
 224 dpdk_ipsec_check_support (ipsec_sa_t * sa)
 225 {
 226   if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
 227     {
 228       if (sa->integ_alg != IPSEC_INTEG_ALG_NONE)
 229         return clib_error_return (0, "unsupported integ-alg %U with "
 230                                   "crypto-alg aes-gcm-128",
 231                                   format_ipsec_integ_alg, sa->integ_alg);
 232 #if DPDK_NO_AEAD
 233       sa->integ_alg = IPSEC_INTEG_ALG_AES_GCM_128;
 234 #endif
 235     }
 236 #if DPDK_NO_AEAD
 237   else if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE ||
 238            sa->integ_alg == IPSEC_INTEG_ALG_NONE ||
 239            sa->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128)
 240 #else
 241   else if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
 242 #endif
 243     return clib_error_return (0,
 244                               "unsupported integ-alg %U with crypto-alg %U",
 245                               format_ipsec_integ_alg, sa->integ_alg,
 246                               format_ipsec_crypto_alg, sa->crypto_alg);
 247
 248   return 0;
 249 }
 250
 251 static uword
 252 dpdk_ipsec_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 253                     vlib_frame_t * f)
 254 {
 255   ipsec_main_t *im = &ipsec_main;
 256   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
 257   vlib_thread_main_t *tm = vlib_get_thread_main ();
 258   struct rte_cryptodev_config dev_conf;
 259   struct rte_cryptodev_qp_conf qp_conf;
 260   struct rte_cryptodev_info cdev_info;
 261   struct rte_mempool *rmp;
 262   i32 dev_id, ret;
 263   u32 i, skip_master;
 264 #if ! DPDK_NO_AEAD
 265   u32 max_sess_size = 0, sess_size;
 266   i8 socket_id;
 267 #endif
 268
 269   if (check_cryptodev_queues () < 0)
 270     {
 271       clib_warning ("not enough Cryptodevs, default to OpenSSL IPsec");
 272       return 0;
 273     }
 274   dcm->enabled = 1;
 275
 276   vec_alloc (dcm->workers_main, tm->n_vlib_mains);
 277   _vec_len (dcm->workers_main) = tm->n_vlib_mains;
 278
 279   fprintf (stdout, "DPDK Cryptodevs info:\n");
 280   fprintf (stdout, "dev_id\tn_qp\tnb_obj\tcache_size\n");
 281   /* HW cryptodevs have higher dev_id, use HW first */
 282   for (dev_id = rte_cryptodev_count () - 1; dev_id >= 0; dev_id--)
 283     {
 284       u16 max_nb_qp, qp = 0;
 285       skip_master = vlib_num_workers () > 0;
 286
 287       rte_cryptodev_info_get (dev_id, &cdev_info);
 288
 289       if (!
 290           (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
 291         continue;
 292
 293       max_nb_qp = cdev_info.max_nb_queue_pairs;
 294
 295       for (i = 0; i < tm->n_vlib_mains; i++)
 296         {
 297           u8 is_outbound;
 298           crypto_worker_main_t *cwm;
 299           uword *map;
 300
 301           if (skip_master)
 302             {
 303               skip_master = 0;
 304               continue;
 305             }
 306
 307           cwm = vec_elt_at_index (dcm->workers_main, i);
 308           map = cwm->algo_qp_map;
 309
 310           if (!map)
 311             {
 312               map = hash_create (0, sizeof (crypto_worker_qp_key_t));
 313               if (!map)
 314                 {
 315                   clib_warning ("unable to create hash table for worker %u",
 316                                 vlib_mains[i]->thread_index);
 317                   goto error;
 318                 }
 319               cwm->algo_qp_map = map;
 320             }
 321
 322           for (is_outbound = 0; is_outbound < 2 && qp < max_nb_qp;
 323                is_outbound++)
 324             qp += add_cdev_mapping (cwm, &cdev_info, dev_id, qp, is_outbound);
 325         }
 326
 327       if (qp == 0)
 328         continue;
 329
 330       dev_conf.socket_id = rte_cryptodev_socket_id (dev_id);
 331       dev_conf.nb_queue_pairs = cdev_info.max_nb_queue_pairs;
 332 #if DPDK_NO_AEAD
 333       dev_conf.session_mp.nb_objs = DPDK_CRYPTO_NB_SESS_OBJS;
 334       dev_conf.session_mp.cache_size = DPDK_CRYPTO_CACHE_SIZE;
 335 #endif
 336       ret = rte_cryptodev_configure (dev_id, &dev_conf);
 337       if (ret < 0)
 338         {
 339           clib_warning ("cryptodev %u config error", dev_id);
 340           goto error;
 341         }
 342
 343       qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
 344       for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++)
 345         {
 346 #if DPDK_NO_AEAD
 347           ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf,
 348                                                 dev_conf.socket_id);
 349 #else
 350           ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf,
 351                                                 dev_conf.socket_id, NULL);
 352 #endif
 353           if (ret < 0)
 354             {
 355               clib_warning ("cryptodev %u qp %u setup error", dev_id, qp);
 356               goto error;
 357             }
 358         }
 359       vec_validate (dcm->cop_pools, dev_conf.socket_id);
 360
 361 #if ! DPDK_NO_AEAD
 362       sess_size = rte_cryptodev_get_private_session_size (dev_id);
 363       if (sess_size > max_sess_size)
 364         max_sess_size = sess_size;
 365 #endif
 366
 367       if (!vec_elt (dcm->cop_pools, dev_conf.socket_id))
 368         {
 369           u8 *pool_name = format (0, "crypto_op_pool_socket%u%c",
 370                                   dev_conf.socket_id, 0);
 371
 372           rmp = rte_crypto_op_pool_create ((char *) pool_name,
 373                                            RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 374                                            DPDK_CRYPTO_NB_COPS *
 375                                            (1 + vlib_num_workers ()),
 376                                            DPDK_CRYPTO_CACHE_SIZE,
 377                                            DPDK_CRYPTO_PRIV_SIZE,
 378                                            dev_conf.socket_id);
 379
 380           if (!rmp)
 381             {
 382               clib_warning ("failed to allocate %s", pool_name);
 383               vec_free (pool_name);
 384               goto error;
 385             }
 386           vec_free (pool_name);
 387           vec_elt (dcm->cop_pools, dev_conf.socket_id) = rmp;
 388         }
 389
 390       fprintf (stdout, "%u\t%u\t%u\t%u\n", dev_id, dev_conf.nb_queue_pairs,
 391                DPDK_CRYPTO_NB_SESS_OBJS, DPDK_CRYPTO_CACHE_SIZE);
 392     }
 393
 394 #if ! DPDK_NO_AEAD
 395   /* *INDENT-OFF* */
 396   vec_foreach_index (socket_id, dcm->cop_pools)
 397     {
 398       u8 *pool_name;
 399
 400       if (!vec_elt (dcm->cop_pools, socket_id))
 401         continue;
 402
 403       vec_validate (dcm->sess_h_pools, socket_id);
 404       pool_name = format (0, "crypto_sess_h_socket%u%c",
 405                               socket_id, 0);
 406       rmp =
 407         rte_mempool_create((i8 *)pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
 408                            rte_cryptodev_get_header_session_size (),
 409                            512, 0, NULL, NULL, NULL, NULL,
 410                            socket_id, 0);
 411       if (!rmp)
 412         {
 413           clib_warning ("failed to allocate %s", pool_name);
 414           vec_free (pool_name);
 415           goto error;
 416         }
 417       vec_free (pool_name);
 418       vec_elt (dcm->sess_h_pools, socket_id) = rmp;
 419
 420       vec_validate (dcm->sess_pools, socket_id);
 421       pool_name = format (0, "crypto_sess_socket%u%c",
 422                               socket_id, 0);
 423       rmp =
 424         rte_mempool_create((i8 *)pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
 425                            max_sess_size, 512, 0, NULL, NULL, NULL, NULL,
 426                            socket_id, 0);
 427       if (!rmp)
 428         {
 429           clib_warning ("failed to allocate %s", pool_name);
 430           vec_free (pool_name);
 431           goto error;
 432         }
 433       vec_free (pool_name);
 434       vec_elt (dcm->sess_pools, socket_id) = rmp;
 435     }
 436   /* *INDENT-ON* */
 437 #endif
 438
 439   dpdk_esp_init ();
 440
 441   /* Add new next node and set as default */
 442   vlib_node_t *node, *next_node;
 443
 444   next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-encrypt");
 445   ASSERT (next_node);
 446   node = vlib_get_node_by_name (vm, (u8 *) "ipsec-output-ip4");
 447   ASSERT (node);
 448   im->esp_encrypt_node_index = next_node->index;
 449   im->esp_encrypt_next_index =
 450     vlib_node_add_next (vm, node->index, next_node->index);
 451
 452   next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-decrypt");
 453   ASSERT (next_node);
 454   node = vlib_get_node_by_name (vm, (u8 *) "ipsec-input-ip4");
 455   ASSERT (node);
 456   im->esp_decrypt_node_index = next_node->index;
 457   im->esp_decrypt_next_index =
 458     vlib_node_add_next (vm, node->index, next_node->index);
 459
 460   im->cb.check_support_cb = dpdk_ipsec_check_support;
 461   im->cb.add_del_sa_sess_cb = add_del_sa_sess;
 462
 463   for (i = 1; i < tm->n_vlib_mains; i++)
 464     vlib_node_set_state (vlib_mains[i], dpdk_crypto_input_node.index,
 465                          VLIB_NODE_STATE_POLLING);
 466
 467   /* TODO cryptodev counters */
 468
 469   return 0;
 470
 471 error:
 472   ;
 473   crypto_worker_main_t *cwm;
 474   struct rte_mempool **mp;
 475   /* *INDENT-OFF* */
 476   vec_foreach (cwm, dcm->workers_main)
 477     hash_free (cwm->algo_qp_map);
 478
 479   vec_foreach (mp, dcm->cop_pools)
 480     {
 481       if (mp)
 482         rte_mempool_free (mp[0]);
 483     }
 484   /* *INDENT-ON* */
 485   vec_free (dcm->workers_main);
 486   vec_free (dcm->cop_pools);
 487
 488   return 0;
 489 }
 490
 491 /* *INDENT-OFF* */
 492 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
 493     .function = dpdk_ipsec_process,
 494     .type = VLIB_NODE_TYPE_PROCESS,
 495     .name = "dpdk-ipsec-process",
 496     .process_log2_n_stack_bytes = 17,
 497 };
 498 /* *INDENT-ON* */
 499
 500 /*
 501  * fd.io coding-style-patch-verification: ON
 502  *
 503  * Local Variables:
 504  * eval: (c-set-style "gnu")
 505  * End:
 506  */