src/plugins/dpdk/ipsec/ipsec.c

   1 /*
   2  * Copyright (c) 2016 Intel and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <vnet/vnet.h>
  16 #include <vnet/ip/ip.h>
  17 #include <vnet/api_errno.h>
  18 #include <vnet/ipsec/ipsec.h>
  19 #include <vlib/node_funcs.h>
  20
  21 #include <dpdk/device/dpdk.h>
  22 #include <dpdk/ipsec/ipsec.h>
  23 #include <dpdk/ipsec/esp.h>
  24
  25 #define DPDK_CRYPTO_NB_SESS_OBJS  20000
  26 #define DPDK_CRYPTO_CACHE_SIZE    512
  27 #define DPDK_CRYPTO_PRIV_SIZE     128
  28 #define DPDK_CRYPTO_N_QUEUE_DESC  1024
  29 #define DPDK_CRYPTO_NB_COPS       (1024 * 4)
  30
  31 static int
  32 add_del_sa_sess (u32 sa_index, u8 is_add)
  33 {
  34   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
  35   crypto_worker_main_t *cwm;
  36   u8 skip_master = vlib_num_workers () > 0;
  37
  38   /* *INDENT-OFF* */
  39   vec_foreach (cwm, dcm->workers_main)
  40     {
  41       crypto_sa_session_t *sa_sess;
  42       u8 is_outbound;
  43
  44       if (skip_master)
  45         {
  46           skip_master = 0;
  47           continue;
  48         }
  49
  50       for (is_outbound = 0; is_outbound < 2; is_outbound++)
  51         {
  52           if (is_add)
  53             {
  54               pool_get (cwm->sa_sess_d[is_outbound], sa_sess);
  55             }
  56           else
  57             {
  58               u8 dev_id;
  59               i32 ret;
  60
  61               sa_sess = pool_elt_at_index (cwm->sa_sess_d[is_outbound], sa_index);
  62               dev_id = cwm->qp_data[sa_sess->qp_index].dev_id;
  63
  64               if (!sa_sess->sess)
  65                 continue;
  66 #if DPDK_NO_AEAD
  67               ret = (rte_cryptodev_sym_session_free(dev_id, sa_sess->sess) == NULL);
  68               ASSERT (ret);
  69 #else
  70               ret = rte_cryptodev_sym_session_clear(dev_id, sa_sess->sess);
  71               ASSERT (!ret);
  72
  73               ret = rte_cryptodev_sym_session_free(sa_sess->sess);
  74               ASSERT (!ret);
  75 #endif
  76               memset(sa_sess, 0, sizeof(sa_sess[0]));
  77             }
  78         }
  79     }
  80   /* *INDENT-OFF* */
  81
  82   return 0;
  83 }
  84
  85 static void
  86 update_qp_data (crypto_worker_main_t * cwm,
  87                 u8 cdev_id, u16 qp_id, u8 is_outbound, u16 * idx)
  88 {
  89   crypto_qp_data_t *qpd;
  90
  91   /* *INDENT-OFF* */
  92   vec_foreach_index (*idx, cwm->qp_data)
  93     {
  94       qpd = vec_elt_at_index(cwm->qp_data, *idx);
  95
  96       if (qpd->dev_id == cdev_id && qpd->qp_id == qp_id &&
  97           qpd->is_outbound == is_outbound)
  98           return;
  99     }
 100   /* *INDENT-ON* */
 101
 102   vec_add2_aligned (cwm->qp_data, qpd, 1, CLIB_CACHE_LINE_BYTES);
 103
 104   qpd->dev_id = cdev_id;
 105   qpd->qp_id = qp_id;
 106   qpd->is_outbound = is_outbound;
 107 }
 108
 109 /*
 110  * return:
 111  *      0: already exist
 112  *      1: mapped
 113  */
 114 static int
 115 add_mapping (crypto_worker_main_t * cwm,
 116              u8 cdev_id, u16 qp, u8 is_outbound,
 117              const struct rte_cryptodev_capabilities *cipher_cap,
 118              const struct rte_cryptodev_capabilities *auth_cap)
 119 {
 120   u16 qp_index;
 121   uword key = 0, data, *ret;
 122   crypto_worker_qp_key_t *p_key = (crypto_worker_qp_key_t *) & key;
 123
 124   p_key->cipher_algo = (u8) cipher_cap->sym.cipher.algo;
 125   p_key->auth_algo = (u8) auth_cap->sym.auth.algo;
 126   p_key->is_outbound = is_outbound;
 127 #if ! DPDK_NO_AEAD
 128   p_key->is_aead = cipher_cap->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD;
 129 #endif
 130
 131   ret = hash_get (cwm->algo_qp_map, key);
 132   if (ret)
 133     return 0;
 134
 135   update_qp_data (cwm, cdev_id, qp, is_outbound, &qp_index);
 136
 137   data = (uword) qp_index;
 138   hash_set (cwm->algo_qp_map, key, data);
 139
 140   return 1;
 141 }
 142
 143 /*
 144  * return:
 145  *      0: already exist
 146  *      1: mapped
 147  */
 148 static int
 149 add_cdev_mapping (crypto_worker_main_t * cwm,
 150                   struct rte_cryptodev_info *dev_info, u8 cdev_id,
 151                   u16 qp, u8 is_outbound)
 152 {
 153   const struct rte_cryptodev_capabilities *i, *j;
 154   u32 mapped = 0;
 155
 156   for (i = dev_info->capabilities; i->op != RTE_CRYPTO_OP_TYPE_UNDEFINED; i++)
 157     {
 158 #if ! DPDK_NO_AEAD
 159       if (i->sym.xform_type == RTE_CRYPTO_SYM_XFORM_AEAD)
 160         {
 161           struct rte_cryptodev_capabilities none = { 0 };
 162
 163           if (check_algo_is_supported (i, NULL) != 0)
 164             continue;
 165
 166           none.sym.auth.algo = RTE_CRYPTO_AUTH_NULL;
 167
 168           mapped |= add_mapping (cwm, cdev_id, qp, is_outbound, i, &none);
 169           continue;
 170         }
 171 #endif
 172       if (i->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
 173         continue;
 174
 175       if (check_algo_is_supported (i, NULL) != 0)
 176         continue;
 177
 178       for (j = dev_info->capabilities; j->op != RTE_CRYPTO_OP_TYPE_UNDEFINED;
 179            j++)
 180         {
 181           if (j->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
 182             continue;
 183
 184           if (check_algo_is_supported (j, NULL) != 0)
 185             continue;
 186
 187           mapped |= add_mapping (cwm, cdev_id, qp, is_outbound, i, j);
 188         }
 189     }
 190
 191   return mapped;
 192 }
 193
 194 static int
 195 check_cryptodev_queues ()
 196 {
 197   u32 n_qs = 0;
 198   u8 cdev_id;
 199   u32 n_req_qs = 2;
 200
 201   if (vlib_num_workers () > 0)
 202     n_req_qs = vlib_num_workers () * 2;
 203
 204   for (cdev_id = 0; cdev_id < rte_cryptodev_count (); cdev_id++)
 205     {
 206       struct rte_cryptodev_info cdev_info;
 207
 208       rte_cryptodev_info_get (cdev_id, &cdev_info);
 209
 210       if (!
 211           (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
 212         continue;
 213
 214       n_qs += cdev_info.max_nb_queue_pairs;
 215     }
 216
 217   if (n_qs >= n_req_qs)
 218     return 0;
 219   else
 220     return -1;
 221 }
 222
 223 static clib_error_t *
 224 dpdk_ipsec_check_support (ipsec_sa_t * sa)
 225 {
 226   if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_GCM_128)
 227     {
 228       if (sa->integ_alg != IPSEC_INTEG_ALG_NONE)
 229         return clib_error_return (0, "unsupported integ-alg %U with "
 230                                   "crypto-alg aes-gcm-128",
 231                                   format_ipsec_integ_alg, sa->integ_alg);
 232 #if DPDK_NO_AEAD
 233       sa->integ_alg = IPSEC_INTEG_ALG_AES_GCM_128;
 234 #endif
 235     }
 236 #if DPDK_NO_AEAD
 237   else if (sa->crypto_alg == IPSEC_CRYPTO_ALG_NONE ||
 238            sa->integ_alg == IPSEC_INTEG_ALG_NONE ||
 239            sa->integ_alg == IPSEC_INTEG_ALG_AES_GCM_128)
 240 #else
 241   else if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
 242 #endif
 243     return clib_error_return (0,
 244                               "unsupported integ-alg %U with crypto-alg %U",
 245                               format_ipsec_integ_alg, sa->integ_alg,
 246                               format_ipsec_crypto_alg, sa->crypto_alg);
 247
 248   return 0;
 249 }
 250
 251 static uword
 252 dpdk_ipsec_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
 253                     vlib_frame_t * f)
 254 {
 255   ipsec_main_t *im = &ipsec_main;
 256   dpdk_crypto_main_t *dcm = &dpdk_crypto_main;
 257   vlib_thread_main_t *tm = vlib_get_thread_main ();
 258   struct rte_cryptodev_config dev_conf;
 259   struct rte_cryptodev_qp_conf qp_conf;
 260   struct rte_cryptodev_info cdev_info;
 261   struct rte_mempool *rmp;
 262   i32 dev_id, ret;
 263   u32 i, skip_master;
 264 #if ! DPDK_NO_AEAD
 265   u32 max_sess_size = 0, sess_size;
 266   i8 socket_id;
 267 #endif
 268
 269   if (check_cryptodev_queues () < 0)
 270     {
 271       clib_warning ("not enough Cryptodevs, default to OpenSSL IPsec");
 272       return 0;
 273     }
 274   dcm->enabled = 1;
 275
 276   vec_alloc (dcm->workers_main, tm->n_vlib_mains);
 277   _vec_len (dcm->workers_main) = tm->n_vlib_mains;
 278
 279   skip_master = vlib_num_workers () > 0;
 280
 281   fprintf (stdout, "DPDK Cryptodevs info:\n");
 282   fprintf (stdout, "dev_id\tn_qp\tnb_obj\tcache_size\n");
 283   /* HW cryptodevs have higher dev_id, use HW first */
 284   for (dev_id = rte_cryptodev_count () - 1; dev_id >= 0; dev_id--)
 285     {
 286       u16 max_nb_qp, qp = 0;
 287
 288       rte_cryptodev_info_get (dev_id, &cdev_info);
 289
 290       if (!
 291           (cdev_info.feature_flags & RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING))
 292         continue;
 293
 294       max_nb_qp = cdev_info.max_nb_queue_pairs;
 295
 296       for (i = 0; i < tm->n_vlib_mains; i++)
 297         {
 298           u8 is_outbound;
 299           crypto_worker_main_t *cwm;
 300           uword *map;
 301
 302           if (skip_master)
 303             {
 304               skip_master = 0;
 305               continue;
 306             }
 307
 308           cwm = vec_elt_at_index (dcm->workers_main, i);
 309           map = cwm->algo_qp_map;
 310
 311           if (!map)
 312             {
 313               map = hash_create (0, sizeof (crypto_worker_qp_key_t));
 314               if (!map)
 315                 {
 316                   clib_warning ("unable to create hash table for worker %u",
 317                                 vlib_mains[i]->thread_index);
 318                   goto error;
 319                 }
 320               cwm->algo_qp_map = map;
 321             }
 322
 323           for (is_outbound = 0; is_outbound < 2 && qp < max_nb_qp;
 324                is_outbound++)
 325             qp += add_cdev_mapping (cwm, &cdev_info, dev_id, qp, is_outbound);
 326         }
 327
 328       if (qp == 0)
 329         continue;
 330
 331       dev_conf.socket_id = rte_cryptodev_socket_id (dev_id);
 332       dev_conf.nb_queue_pairs = cdev_info.max_nb_queue_pairs;
 333 #if DPDK_NO_AEAD
 334       dev_conf.session_mp.nb_objs = DPDK_CRYPTO_NB_SESS_OBJS;
 335       dev_conf.session_mp.cache_size = DPDK_CRYPTO_CACHE_SIZE;
 336 #endif
 337       ret = rte_cryptodev_configure (dev_id, &dev_conf);
 338       if (ret < 0)
 339         {
 340           clib_warning ("cryptodev %u config error", dev_id);
 341           goto error;
 342         }
 343
 344       qp_conf.nb_descriptors = DPDK_CRYPTO_N_QUEUE_DESC;
 345       for (qp = 0; qp < dev_conf.nb_queue_pairs; qp++)
 346         {
 347 #if DPDK_NO_AEAD
 348           ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf,
 349                                                 dev_conf.socket_id);
 350 #else
 351           ret = rte_cryptodev_queue_pair_setup (dev_id, qp, &qp_conf,
 352                                                 dev_conf.socket_id, NULL);
 353 #endif
 354           if (ret < 0)
 355             {
 356               clib_warning ("cryptodev %u qp %u setup error", dev_id, qp);
 357               goto error;
 358             }
 359         }
 360       vec_validate (dcm->cop_pools, dev_conf.socket_id);
 361
 362 #if ! DPDK_NO_AEAD
 363       sess_size = rte_cryptodev_get_private_session_size (dev_id);
 364       if (sess_size > max_sess_size)
 365         max_sess_size = sess_size;
 366 #endif
 367
 368       if (!vec_elt (dcm->cop_pools, dev_conf.socket_id))
 369         {
 370           u8 *pool_name = format (0, "crypto_op_pool_socket%u%c",
 371                                   dev_conf.socket_id, 0);
 372
 373           rmp = rte_crypto_op_pool_create ((char *) pool_name,
 374                                            RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 375                                            DPDK_CRYPTO_NB_COPS *
 376                                            (1 + vlib_num_workers ()),
 377                                            DPDK_CRYPTO_CACHE_SIZE,
 378                                            DPDK_CRYPTO_PRIV_SIZE,
 379                                            dev_conf.socket_id);
 380
 381           if (!rmp)
 382             {
 383               clib_warning ("failed to allocate %s", pool_name);
 384               vec_free (pool_name);
 385               goto error;
 386             }
 387           vec_free (pool_name);
 388           vec_elt (dcm->cop_pools, dev_conf.socket_id) = rmp;
 389         }
 390
 391       fprintf (stdout, "%u\t%u\t%u\t%u\n", dev_id, dev_conf.nb_queue_pairs,
 392                DPDK_CRYPTO_NB_SESS_OBJS, DPDK_CRYPTO_CACHE_SIZE);
 393     }
 394
 395 #if ! DPDK_NO_AEAD
 396   /* *INDENT-OFF* */
 397   vec_foreach_index (socket_id, dcm->cop_pools)
 398     {
 399       u8 *pool_name;
 400
 401       if (!vec_elt (dcm->cop_pools, socket_id))
 402         continue;
 403
 404       vec_validate (dcm->sess_h_pools, socket_id);
 405       pool_name = format (0, "crypto_sess_h_socket%u%c",
 406                               socket_id, 0);
 407       rmp =
 408         rte_mempool_create((i8 *)pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
 409                            rte_cryptodev_get_header_session_size (),
 410                            512, 0, NULL, NULL, NULL, NULL,
 411                            socket_id, 0);
 412       if (!rmp)
 413         {
 414           clib_warning ("failed to allocate %s", pool_name);
 415           vec_free (pool_name);
 416           goto error;
 417         }
 418       vec_free (pool_name);
 419       vec_elt (dcm->sess_h_pools, socket_id) = rmp;
 420
 421       vec_validate (dcm->sess_pools, socket_id);
 422       pool_name = format (0, "crypto_sess_socket%u%c",
 423                               socket_id, 0);
 424       rmp =
 425         rte_mempool_create((i8 *)pool_name, DPDK_CRYPTO_NB_SESS_OBJS,
 426                            max_sess_size, 512, 0, NULL, NULL, NULL, NULL,
 427                            socket_id, 0);
 428       if (!rmp)
 429         {
 430           clib_warning ("failed to allocate %s", pool_name);
 431           vec_free (pool_name);
 432           goto error;
 433         }
 434       vec_free (pool_name);
 435       vec_elt (dcm->sess_pools, socket_id) = rmp;
 436     }
 437   /* *INDENT-ON* */
 438 #endif
 439
 440   dpdk_esp_init ();
 441
 442   /* Add new next node and set as default */
 443   vlib_node_t *node, *next_node;
 444
 445   next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-encrypt");
 446   ASSERT (next_node);
 447   node = vlib_get_node_by_name (vm, (u8 *) "ipsec-output-ip4");
 448   ASSERT (node);
 449   im->esp_encrypt_node_index = next_node->index;
 450   im->esp_encrypt_next_index =
 451     vlib_node_add_next (vm, node->index, next_node->index);
 452
 453   next_node = vlib_get_node_by_name (vm, (u8 *) "dpdk-esp-decrypt");
 454   ASSERT (next_node);
 455   node = vlib_get_node_by_name (vm, (u8 *) "ipsec-input-ip4");
 456   ASSERT (node);
 457   im->esp_decrypt_node_index = next_node->index;
 458   im->esp_decrypt_next_index =
 459     vlib_node_add_next (vm, node->index, next_node->index);
 460
 461   im->cb.check_support_cb = dpdk_ipsec_check_support;
 462   im->cb.add_del_sa_sess_cb = add_del_sa_sess;
 463
 464   for (i = skip_master; i < tm->n_vlib_mains; i++)
 465     vlib_node_set_state (vlib_mains[i], dpdk_crypto_input_node.index,
 466                          VLIB_NODE_STATE_POLLING);
 467
 468   /* TODO cryptodev counters */
 469
 470   return 0;
 471
 472 error:
 473   ;
 474   crypto_worker_main_t *cwm;
 475   struct rte_mempool **mp;
 476   /* *INDENT-OFF* */
 477   vec_foreach (cwm, dcm->workers_main)
 478     hash_free (cwm->algo_qp_map);
 479
 480   vec_foreach (mp, dcm->cop_pools)
 481     {
 482       if (mp)
 483         rte_mempool_free (mp[0]);
 484     }
 485   /* *INDENT-ON* */
 486   vec_free (dcm->workers_main);
 487   vec_free (dcm->cop_pools);
 488
 489   return 0;
 490 }
 491
 492 /* *INDENT-OFF* */
 493 VLIB_REGISTER_NODE (dpdk_ipsec_process_node,static) = {
 494     .function = dpdk_ipsec_process,
 495     .type = VLIB_NODE_TYPE_PROCESS,
 496     .name = "dpdk-ipsec-process",
 497     .process_log2_n_stack_bytes = 17,
 498 };
 499 /* *INDENT-ON* */
 500
 501 /*
 502  * fd.io coding-style-patch-verification: ON
 503  *
 504  * Local Variables:
 505  * eval: (c-set-style "gnu")
 506  * End:
 507  */