src/vnet/ipsec/ipsec.c

   1 /*
   2  * ipsec.c : IPSEC module functions
   3  *
   4  * Copyright (c) 2015 Cisco and/or its affiliates.
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <vnet/vnet.h>
  19 #include <vnet/api_errno.h>
  20 #include <vnet/ip/ip.h>
  21 #include <vnet/interface.h>
  22 #include <vnet/udp/udp_local.h>
  23
  24 #include <vnet/ipsec/ipsec.h>
  25 #include <vnet/ipsec/esp.h>
  26 #include <vnet/ipsec/ah.h>
  27 #include <vnet/ipsec/ipsec_tun.h>
  28 #include <vnet/ipsec/ipsec_itf.h>
  29 #include <vnet/ipsec/ipsec_spd_fp_lookup.h>
  30
  31 /* Flow cache is sized for 1 million flows with a load factor of .25.
  32  */
  33 #define IPSEC4_OUT_SPD_DEFAULT_HASH_NUM_BUCKETS (1 << 22)
  34
  35 /* Flow cache is sized for 1 million flows with a load factor of .25.
  36  */
  37 #define IPSEC4_SPD_DEFAULT_HASH_NUM_BUCKETS (1 << 22)
  38
  39 ipsec_main_t ipsec_main;
  40
  41 esp_async_post_next_t esp_encrypt_async_next;
  42 esp_async_post_next_t esp_decrypt_async_next;
  43
  44 clib_error_t *
  45 ipsec_register_next_header (vlib_main_t *vm, u8 next_header,
  46                             const char *next_node)
  47 {
  48   ipsec_main_t *im = &ipsec_main;
  49   const vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) next_node);
  50   /* -post nodes (eg. esp4-decrypt-post) are siblings of non-post nodes (eg.
  51    * esp4-decrypt) and will therefore have the same next index */
  52   const vlib_node_t *esp_decrypt_nodes[] = {
  53     vlib_get_node (vm, im->esp4_decrypt_node_index),
  54     vlib_get_node (vm, im->esp6_decrypt_node_index),
  55     vlib_get_node (vm, im->esp4_decrypt_tun_node_index),
  56     vlib_get_node (vm, im->esp6_decrypt_tun_node_index),
  57   };
  58   uword slot, max;
  59   int i;
  60
  61   /* looks for a next_index value that we can use for all esp decrypt nodes to
  62    * avoid maintaining different next index arrays... */
  63
  64   slot = vlib_node_get_next (vm, esp_decrypt_nodes[0]->index, node->index);
  65   max = vec_len (esp_decrypt_nodes[0]->next_nodes);
  66   for (i = 1; i < ARRAY_LEN (esp_decrypt_nodes); i++)
  67     {
  68       /* if next node already exists, check it shares the same next_index */
  69       if (slot !=
  70           vlib_node_get_next (vm, esp_decrypt_nodes[i]->index, node->index))
  71         return clib_error_return (
  72           0, "next node already exists with different next index");
  73       /* compute a suitable slot from the max of all nodes next index */
  74       max = clib_max (max, vec_len (esp_decrypt_nodes[i]->next_nodes));
  75     }
  76
  77   if (~0 == slot)
  78     {
  79       /* next node not there yet, add it using the computed max */
  80       slot = max;
  81       for (i = 0; i < ARRAY_LEN (esp_decrypt_nodes); i++)
  82         vlib_node_add_next_with_slot (vm, esp_decrypt_nodes[i]->index,
  83                                       node->index, slot);
  84     }
  85
  86   im->next_header_registrations[next_header] = slot;
  87   return 0;
  88 }
  89
  90 static clib_error_t *
  91 ipsec_check_ah_support (ipsec_sa_t * sa)
  92 {
  93   ipsec_main_t *im = &ipsec_main;
  94
  95   if (sa->integ_alg == IPSEC_INTEG_ALG_NONE)
  96     return clib_error_return (0, "unsupported none integ-alg");
  97
  98   if (!vnet_crypto_is_set_handler (im->integ_algs[sa->integ_alg].alg))
  99     return clib_error_return (0, "No crypto engine support for %U",
 100                               format_ipsec_integ_alg, sa->integ_alg);
 101
 102   return 0;
 103 }
 104
 105 static clib_error_t *
 106 ipsec_check_esp_support (ipsec_sa_t * sa)
 107 {
 108   ipsec_main_t *im = &ipsec_main;
 109
 110   if (IPSEC_INTEG_ALG_NONE != sa->integ_alg)
 111     {
 112       if (!vnet_crypto_is_set_handler (im->integ_algs[sa->integ_alg].alg))
 113         return clib_error_return (0, "No crypto engine support for %U",
 114                                   format_ipsec_integ_alg, sa->integ_alg);
 115     }
 116   if (IPSEC_CRYPTO_ALG_NONE != sa->crypto_alg)
 117     {
 118       if (!vnet_crypto_is_set_handler (im->crypto_algs[sa->crypto_alg].alg))
 119         return clib_error_return (0, "No crypto engine support for %U",
 120                                   format_ipsec_crypto_alg, sa->crypto_alg);
 121     }
 122
 123   return (0);
 124 }
 125
 126 clib_error_t *
 127 ipsec_add_del_sa_sess_cb (ipsec_main_t * im, u32 sa_index, u8 is_add)
 128 {
 129   ipsec_ah_backend_t *ah =
 130     pool_elt_at_index (im->ah_backends, im->ah_current_backend);
 131   if (ah->add_del_sa_sess_cb)
 132     {
 133       clib_error_t *err = ah->add_del_sa_sess_cb (sa_index, is_add);
 134       if (err)
 135         return err;
 136     }
 137   ipsec_esp_backend_t *esp =
 138     pool_elt_at_index (im->esp_backends, im->esp_current_backend);
 139   if (esp->add_del_sa_sess_cb)
 140     {
 141       clib_error_t *err = esp->add_del_sa_sess_cb (sa_index, is_add);
 142       if (err)
 143         return err;
 144     }
 145   return 0;
 146 }
 147
 148 clib_error_t *
 149 ipsec_check_support_cb (ipsec_main_t * im, ipsec_sa_t * sa)
 150 {
 151   clib_error_t *error = 0;
 152
 153   if (PREDICT_FALSE (sa->protocol == IPSEC_PROTOCOL_AH))
 154     {
 155       ipsec_ah_backend_t *ah =
 156         pool_elt_at_index (im->ah_backends, im->ah_current_backend);
 157       ASSERT (ah->check_support_cb);
 158       error = ah->check_support_cb (sa);
 159     }
 160   else
 161     {
 162       ipsec_esp_backend_t *esp =
 163         pool_elt_at_index (im->esp_backends, im->esp_current_backend);
 164       ASSERT (esp->check_support_cb);
 165       error = esp->check_support_cb (sa);
 166     }
 167   return error;
 168 }
 169
 170
 171 static void
 172 ipsec_add_node (vlib_main_t * vm, const char *node_name,
 173                 const char *prev_node_name, u32 * out_node_index,
 174                 u32 * out_next_index)
 175 {
 176   vlib_node_t *prev_node, *node;
 177   prev_node = vlib_get_node_by_name (vm, (u8 *) prev_node_name);
 178   ASSERT (prev_node);
 179   node = vlib_get_node_by_name (vm, (u8 *) node_name);
 180   ASSERT (node);
 181   *out_node_index = node->index;
 182   *out_next_index = vlib_node_add_next (vm, prev_node->index, node->index);
 183 }
 184
 185 static inline uword
 186 ipsec_udp_registration_key (u16 port, u8 is_ip4)
 187 {
 188   uword key = (is_ip4) ? AF_IP4 : AF_IP6;
 189
 190   key |= (uword) (port << 16);
 191   return key;
 192 }
 193
 194 void
 195 ipsec_unregister_udp_port (u16 port, u8 is_ip4)
 196 {
 197   ipsec_main_t *im = &ipsec_main;
 198   u32 n_regs;
 199   uword *p, key;
 200
 201   key = ipsec_udp_registration_key (port, is_ip4);
 202   p = hash_get (im->udp_port_registrations, key);
 203
 204   ASSERT (p);
 205
 206   n_regs = p[0];
 207
 208   if (0 == --n_regs)
 209     {
 210       udp_unregister_dst_port (vlib_get_main (), port, is_ip4);
 211       hash_unset (im->udp_port_registrations, key);
 212     }
 213   else
 214     {
 215       hash_unset (im->udp_port_registrations, key);
 216       hash_set (im->udp_port_registrations, key, n_regs);
 217     }
 218 }
 219
 220 void
 221 ipsec_register_udp_port (u16 port, u8 is_ip4)
 222 {
 223   ipsec_main_t *im = &ipsec_main;
 224   u32 n_regs, node_index;
 225   uword *p, key;
 226
 227   key = ipsec_udp_registration_key (port, is_ip4);
 228   node_index =
 229     (is_ip4) ? ipsec4_tun_input_node.index : ipsec6_tun_input_node.index;
 230   p = hash_get (im->udp_port_registrations, key);
 231
 232   n_regs = (p ? p[0] : 0);
 233
 234   if (0 == n_regs++)
 235     udp_register_dst_port (vlib_get_main (), port, node_index, is_ip4);
 236
 237   hash_unset (im->udp_port_registrations, key);
 238   hash_set (im->udp_port_registrations, key, n_regs);
 239 }
 240
 241 u32
 242 ipsec_register_ah_backend (vlib_main_t * vm, ipsec_main_t * im,
 243                            const char *name,
 244                            const char *ah4_encrypt_node_name,
 245                            const char *ah4_decrypt_node_name,
 246                            const char *ah6_encrypt_node_name,
 247                            const char *ah6_decrypt_node_name,
 248                            check_support_cb_t ah_check_support_cb,
 249                            add_del_sa_sess_cb_t ah_add_del_sa_sess_cb)
 250 {
 251   ipsec_ah_backend_t *b;
 252   pool_get (im->ah_backends, b);
 253   b->name = format (0, "%s%c", name, 0);
 254
 255   ipsec_add_node (vm, ah4_encrypt_node_name, "ipsec4-output-feature",
 256                   &b->ah4_encrypt_node_index, &b->ah4_encrypt_next_index);
 257   ipsec_add_node (vm, ah4_decrypt_node_name, "ipsec4-input-feature",
 258                   &b->ah4_decrypt_node_index, &b->ah4_decrypt_next_index);
 259   ipsec_add_node (vm, ah6_encrypt_node_name, "ipsec6-output-feature",
 260                   &b->ah6_encrypt_node_index, &b->ah6_encrypt_next_index);
 261   ipsec_add_node (vm, ah6_decrypt_node_name, "ipsec6-input-feature",
 262                   &b->ah6_decrypt_node_index, &b->ah6_decrypt_next_index);
 263
 264   b->check_support_cb = ah_check_support_cb;
 265   b->add_del_sa_sess_cb = ah_add_del_sa_sess_cb;
 266   return b - im->ah_backends;
 267 }
 268
 269 u32
 270 ipsec_register_esp_backend (
 271   vlib_main_t *vm, ipsec_main_t *im, const char *name,
 272   const char *esp4_encrypt_node_name, const char *esp4_encrypt_node_tun_name,
 273   const char *esp4_decrypt_node_name, const char *esp4_decrypt_tun_node_name,
 274   const char *esp6_encrypt_node_name, const char *esp6_encrypt_node_tun_name,
 275   const char *esp6_decrypt_node_name, const char *esp6_decrypt_tun_node_name,
 276   const char *esp_mpls_encrypt_node_tun_name,
 277   check_support_cb_t esp_check_support_cb,
 278   add_del_sa_sess_cb_t esp_add_del_sa_sess_cb)
 279 {
 280   ipsec_esp_backend_t *b;
 281
 282   pool_get (im->esp_backends, b);
 283   b->name = format (0, "%s%c", name, 0);
 284
 285   ipsec_add_node (vm, esp4_encrypt_node_name, "ipsec4-output-feature",
 286                   &b->esp4_encrypt_node_index, &b->esp4_encrypt_next_index);
 287   ipsec_add_node (vm, esp4_decrypt_node_name, "ipsec4-input-feature",
 288                   &b->esp4_decrypt_node_index, &b->esp4_decrypt_next_index);
 289   ipsec_add_node (vm, esp6_encrypt_node_name, "ipsec6-output-feature",
 290                   &b->esp6_encrypt_node_index, &b->esp6_encrypt_next_index);
 291   ipsec_add_node (vm, esp6_decrypt_node_name, "ipsec6-input-feature",
 292                   &b->esp6_decrypt_node_index, &b->esp6_decrypt_next_index);
 293   ipsec_add_node (vm, esp4_decrypt_tun_node_name, "ipsec4-tun-input",
 294                   &b->esp4_decrypt_tun_node_index,
 295                   &b->esp4_decrypt_tun_next_index);
 296   ipsec_add_node (vm, esp6_decrypt_tun_node_name, "ipsec6-tun-input",
 297                   &b->esp6_decrypt_tun_node_index,
 298                   &b->esp6_decrypt_tun_next_index);
 299
 300   b->esp6_encrypt_tun_node_index =
 301     vlib_get_node_by_name (vm, (u8 *) esp6_encrypt_node_tun_name)->index;
 302   b->esp_mpls_encrypt_tun_node_index =
 303     vlib_get_node_by_name (vm, (u8 *) esp_mpls_encrypt_node_tun_name)->index;
 304   b->esp4_encrypt_tun_node_index =
 305     vlib_get_node_by_name (vm, (u8 *) esp4_encrypt_node_tun_name)->index;
 306
 307   b->check_support_cb = esp_check_support_cb;
 308   b->add_del_sa_sess_cb = esp_add_del_sa_sess_cb;
 309
 310   return b - im->esp_backends;
 311 }
 312
 313 clib_error_t *
 314 ipsec_rsc_in_use (ipsec_main_t * im)
 315 {
 316   /* return an error is crypto resource are in use */
 317   if (pool_elts (ipsec_sa_pool) > 0)
 318     return clib_error_return (0, "%d SA entries configured",
 319                               pool_elts (ipsec_sa_pool));
 320   if (ipsec_itf_count () > 0)
 321     return clib_error_return (0, "%d IPSec interface configured",
 322                               ipsec_itf_count ());
 323
 324   return (NULL);
 325 }
 326
 327 int
 328 ipsec_select_ah_backend (ipsec_main_t * im, u32 backend_idx)
 329 {
 330   if (ipsec_rsc_in_use (im))
 331     return VNET_API_ERROR_RSRC_IN_USE;
 332
 333   if (pool_is_free_index (im->ah_backends, backend_idx))
 334     return VNET_API_ERROR_INVALID_VALUE;
 335
 336   ipsec_ah_backend_t *b = pool_elt_at_index (im->ah_backends, backend_idx);
 337   im->ah_current_backend = backend_idx;
 338   im->ah4_encrypt_node_index = b->ah4_encrypt_node_index;
 339   im->ah4_decrypt_node_index = b->ah4_decrypt_node_index;
 340   im->ah4_encrypt_next_index = b->ah4_encrypt_next_index;
 341   im->ah4_decrypt_next_index = b->ah4_decrypt_next_index;
 342   im->ah6_encrypt_node_index = b->ah6_encrypt_node_index;
 343   im->ah6_decrypt_node_index = b->ah6_decrypt_node_index;
 344   im->ah6_encrypt_next_index = b->ah6_encrypt_next_index;
 345   im->ah6_decrypt_next_index = b->ah6_decrypt_next_index;
 346
 347   return 0;
 348 }
 349
 350 int
 351 ipsec_select_esp_backend (ipsec_main_t * im, u32 backend_idx)
 352 {
 353   if (ipsec_rsc_in_use (im))
 354     return VNET_API_ERROR_RSRC_IN_USE;
 355
 356   if (pool_is_free_index (im->esp_backends, backend_idx))
 357     return VNET_API_ERROR_INVALID_VALUE;
 358
 359   ipsec_esp_backend_t *b = pool_elt_at_index (im->esp_backends, backend_idx);
 360   im->esp_current_backend = backend_idx;
 361   im->esp4_encrypt_node_index = b->esp4_encrypt_node_index;
 362   im->esp4_decrypt_node_index = b->esp4_decrypt_node_index;
 363   im->esp4_encrypt_next_index = b->esp4_encrypt_next_index;
 364   im->esp4_decrypt_next_index = b->esp4_decrypt_next_index;
 365   im->esp6_encrypt_node_index = b->esp6_encrypt_node_index;
 366   im->esp6_decrypt_node_index = b->esp6_decrypt_node_index;
 367   im->esp6_encrypt_next_index = b->esp6_encrypt_next_index;
 368   im->esp6_decrypt_next_index = b->esp6_decrypt_next_index;
 369   im->esp4_decrypt_tun_node_index = b->esp4_decrypt_tun_node_index;
 370   im->esp4_decrypt_tun_next_index = b->esp4_decrypt_tun_next_index;
 371   im->esp6_decrypt_tun_node_index = b->esp6_decrypt_tun_node_index;
 372   im->esp6_decrypt_tun_next_index = b->esp6_decrypt_tun_next_index;
 373   im->esp4_encrypt_tun_node_index = b->esp4_encrypt_tun_node_index;
 374   im->esp6_encrypt_tun_node_index = b->esp6_encrypt_tun_node_index;
 375   im->esp_mpls_encrypt_tun_node_index = b->esp_mpls_encrypt_tun_node_index;
 376
 377   return 0;
 378 }
 379
 380 void
 381 ipsec_set_async_mode (u32 is_enabled)
 382 {
 383   ipsec_main_t *im = &ipsec_main;
 384   ipsec_sa_t *sa;
 385
 386   im->async_mode = is_enabled;
 387
 388   /* change SA crypto op data */
 389   pool_foreach (sa, ipsec_sa_pool)
 390     ipsec_sa_set_async_mode (sa, is_enabled);
 391 }
 392
 393 static void
 394 crypto_engine_backend_register_post_node (vlib_main_t * vm)
 395 {
 396   esp_async_post_next_t *eit;
 397   esp_async_post_next_t *dit;
 398
 399   eit = &esp_encrypt_async_next;
 400   eit->esp4_post_next =
 401     vnet_crypto_register_post_node (vm, "esp4-encrypt-post");
 402   eit->esp6_post_next =
 403     vnet_crypto_register_post_node (vm, "esp6-encrypt-post");
 404   eit->esp4_tun_post_next =
 405     vnet_crypto_register_post_node (vm, "esp4-encrypt-tun-post");
 406   eit->esp6_tun_post_next =
 407     vnet_crypto_register_post_node (vm, "esp6-encrypt-tun-post");
 408   eit->esp_mpls_tun_post_next =
 409     vnet_crypto_register_post_node (vm, "esp-mpls-encrypt-tun-post");
 410
 411   dit = &esp_decrypt_async_next;
 412   dit->esp4_post_next =
 413     vnet_crypto_register_post_node (vm, "esp4-decrypt-post");
 414   dit->esp6_post_next =
 415     vnet_crypto_register_post_node (vm, "esp6-decrypt-post");
 416   dit->esp4_tun_post_next =
 417     vnet_crypto_register_post_node (vm, "esp4-decrypt-tun-post");
 418   dit->esp6_tun_post_next =
 419     vnet_crypto_register_post_node (vm, "esp6-decrypt-tun-post");
 420 }
 421
 422 static clib_error_t *
 423 ipsec_init (vlib_main_t * vm)
 424 {
 425   clib_error_t *error;
 426   ipsec_main_t *im = &ipsec_main;
 427   ipsec_main_crypto_alg_t *a;
 428
 429   /* Backend registration requires the feature arcs to be set up */
 430   if ((error = vlib_call_init_function (vm, vnet_feature_init)))
 431     return (error);
 432
 433   im->vnet_main = vnet_get_main ();
 434   im->vlib_main = vm;
 435
 436   im->spd_index_by_spd_id = hash_create (0, sizeof (uword));
 437   im->sa_index_by_sa_id = hash_create (0, sizeof (uword));
 438   im->spd_index_by_sw_if_index = hash_create (0, sizeof (uword));
 439
 440   vlib_node_t *node = vlib_get_node_by_name (vm, (u8 *) "error-drop");
 441   ASSERT (node);
 442   im->error_drop_node_index = node->index;
 443
 444   im->ah_current_backend = ~0;
 445   im->esp_current_backend = ~0;
 446
 447   u32 idx = ipsec_register_ah_backend (vm, im, "crypto engine backend",
 448                                        "ah4-encrypt",
 449                                        "ah4-decrypt",
 450                                        "ah6-encrypt",
 451                                        "ah6-decrypt",
 452                                        ipsec_check_ah_support,
 453                                        NULL);
 454
 455   im->ah_default_backend = idx;
 456   int rv = ipsec_select_ah_backend (im, idx);
 457   ASSERT (0 == rv);
 458   (void) (rv);                  // avoid warning
 459
 460   idx = ipsec_register_esp_backend (
 461     vm, im, "crypto engine backend", "esp4-encrypt", "esp4-encrypt-tun",
 462     "esp4-decrypt", "esp4-decrypt-tun", "esp6-encrypt", "esp6-encrypt-tun",
 463     "esp6-decrypt", "esp6-decrypt-tun", "esp-mpls-encrypt-tun",
 464     ipsec_check_esp_support, NULL);
 465   im->esp_default_backend = idx;
 466
 467   rv = ipsec_select_esp_backend (im, idx);
 468   ASSERT (0 == rv);
 469   (void) (rv);                  // avoid warning
 470
 471   if ((error = vlib_call_init_function (vm, ipsec_cli_init)))
 472     return error;
 473
 474   vec_validate (im->crypto_algs, IPSEC_CRYPTO_N_ALG - 1);
 475
 476   a = im->crypto_algs + IPSEC_CRYPTO_ALG_NONE;
 477   a->enc_op_id = VNET_CRYPTO_OP_NONE;
 478   a->dec_op_id = VNET_CRYPTO_OP_NONE;
 479   a->alg = VNET_CRYPTO_ALG_NONE;
 480   a->iv_size = 0;
 481   a->block_align = 1;
 482
 483   a = im->crypto_algs + IPSEC_CRYPTO_ALG_DES_CBC;
 484   a->enc_op_id = VNET_CRYPTO_OP_DES_CBC_ENC;
 485   a->dec_op_id = VNET_CRYPTO_OP_DES_CBC_DEC;
 486   a->alg = VNET_CRYPTO_ALG_DES_CBC;
 487   a->iv_size = a->block_align = 8;
 488
 489   a = im->crypto_algs + IPSEC_CRYPTO_ALG_3DES_CBC;
 490   a->enc_op_id = VNET_CRYPTO_OP_3DES_CBC_ENC;
 491   a->dec_op_id = VNET_CRYPTO_OP_3DES_CBC_DEC;
 492   a->alg = VNET_CRYPTO_ALG_3DES_CBC;
 493   a->iv_size = a->block_align = 8;
 494
 495   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_CBC_128;
 496   a->enc_op_id = VNET_CRYPTO_OP_AES_128_CBC_ENC;
 497   a->dec_op_id = VNET_CRYPTO_OP_AES_128_CBC_DEC;
 498   a->alg = VNET_CRYPTO_ALG_AES_128_CBC;
 499   a->iv_size = a->block_align = 16;
 500
 501   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_CBC_192;
 502   a->enc_op_id = VNET_CRYPTO_OP_AES_192_CBC_ENC;
 503   a->dec_op_id = VNET_CRYPTO_OP_AES_192_CBC_DEC;
 504   a->alg = VNET_CRYPTO_ALG_AES_192_CBC;
 505   a->iv_size = a->block_align = 16;
 506
 507   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_CBC_256;
 508   a->enc_op_id = VNET_CRYPTO_OP_AES_256_CBC_ENC;
 509   a->dec_op_id = VNET_CRYPTO_OP_AES_256_CBC_DEC;
 510   a->alg = VNET_CRYPTO_ALG_AES_256_CBC;
 511   a->iv_size = a->block_align = 16;
 512
 513   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_CTR_128;
 514   a->enc_op_id = VNET_CRYPTO_OP_AES_128_CTR_ENC;
 515   a->dec_op_id = VNET_CRYPTO_OP_AES_128_CTR_DEC;
 516   a->alg = VNET_CRYPTO_ALG_AES_128_CTR;
 517   a->iv_size = 8;
 518   a->block_align = 1;
 519
 520   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_CTR_192;
 521   a->enc_op_id = VNET_CRYPTO_OP_AES_192_CTR_ENC;
 522   a->dec_op_id = VNET_CRYPTO_OP_AES_192_CTR_DEC;
 523   a->alg = VNET_CRYPTO_ALG_AES_192_CTR;
 524   a->iv_size = 8;
 525   a->block_align = 1;
 526
 527   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_CTR_256;
 528   a->enc_op_id = VNET_CRYPTO_OP_AES_256_CTR_ENC;
 529   a->dec_op_id = VNET_CRYPTO_OP_AES_256_CTR_DEC;
 530   a->alg = VNET_CRYPTO_ALG_AES_256_CTR;
 531   a->iv_size = 8;
 532   a->block_align = 1;
 533
 534   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_GCM_128;
 535   a->enc_op_id = VNET_CRYPTO_OP_AES_128_GCM_ENC;
 536   a->dec_op_id = VNET_CRYPTO_OP_AES_128_GCM_DEC;
 537   a->alg = VNET_CRYPTO_ALG_AES_128_GCM;
 538   a->iv_size = 8;
 539   a->block_align = 1;
 540   a->icv_size = 16;
 541
 542   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_GCM_192;
 543   a->enc_op_id = VNET_CRYPTO_OP_AES_192_GCM_ENC;
 544   a->dec_op_id = VNET_CRYPTO_OP_AES_192_GCM_DEC;
 545   a->alg = VNET_CRYPTO_ALG_AES_192_GCM;
 546   a->iv_size = 8;
 547   a->block_align = 1;
 548   a->icv_size = 16;
 549
 550   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_GCM_256;
 551   a->enc_op_id = VNET_CRYPTO_OP_AES_256_GCM_ENC;
 552   a->dec_op_id = VNET_CRYPTO_OP_AES_256_GCM_DEC;
 553   a->alg = VNET_CRYPTO_ALG_AES_256_GCM;
 554   a->iv_size = 8;
 555   a->block_align = 1;
 556   a->icv_size = 16;
 557
 558   a = im->crypto_algs + IPSEC_CRYPTO_ALG_CHACHA20_POLY1305;
 559   a->enc_op_id = VNET_CRYPTO_OP_CHACHA20_POLY1305_ENC;
 560   a->dec_op_id = VNET_CRYPTO_OP_CHACHA20_POLY1305_DEC;
 561   a->alg = VNET_CRYPTO_ALG_CHACHA20_POLY1305;
 562   a->iv_size = 8;
 563   a->icv_size = 16;
 564
 565   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_NULL_GMAC_128;
 566   a->enc_op_id = VNET_CRYPTO_OP_AES_128_NULL_GMAC_ENC;
 567   a->dec_op_id = VNET_CRYPTO_OP_AES_128_NULL_GMAC_DEC;
 568   a->alg = VNET_CRYPTO_ALG_AES_128_GCM;
 569   a->iv_size = 8;
 570   a->block_align = 1;
 571   a->icv_size = 16;
 572
 573   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_NULL_GMAC_192;
 574   a->enc_op_id = VNET_CRYPTO_OP_AES_192_NULL_GMAC_ENC;
 575   a->dec_op_id = VNET_CRYPTO_OP_AES_192_NULL_GMAC_DEC;
 576   a->alg = VNET_CRYPTO_ALG_AES_192_GCM;
 577   a->iv_size = 8;
 578   a->block_align = 1;
 579   a->icv_size = 16;
 580
 581   a = im->crypto_algs + IPSEC_CRYPTO_ALG_AES_NULL_GMAC_256;
 582   a->enc_op_id = VNET_CRYPTO_OP_AES_256_NULL_GMAC_ENC;
 583   a->dec_op_id = VNET_CRYPTO_OP_AES_256_NULL_GMAC_DEC;
 584   a->alg = VNET_CRYPTO_ALG_AES_256_GCM;
 585   a->iv_size = 8;
 586   a->block_align = 1;
 587   a->icv_size = 16;
 588
 589   vec_validate (im->integ_algs, IPSEC_INTEG_N_ALG - 1);
 590   ipsec_main_integ_alg_t *i;
 591
 592   i = &im->integ_algs[IPSEC_INTEG_ALG_MD5_96];
 593   i->op_id = VNET_CRYPTO_OP_MD5_HMAC;
 594   i->alg = VNET_CRYPTO_ALG_HMAC_MD5;
 595   i->icv_size = 12;
 596
 597   i = &im->integ_algs[IPSEC_INTEG_ALG_SHA1_96];
 598   i->op_id = VNET_CRYPTO_OP_SHA1_HMAC;
 599   i->alg = VNET_CRYPTO_ALG_HMAC_SHA1;
 600   i->icv_size = 12;
 601
 602   i = &im->integ_algs[IPSEC_INTEG_ALG_SHA_256_96];
 603   i->op_id = VNET_CRYPTO_OP_SHA1_HMAC;
 604   i->alg = VNET_CRYPTO_ALG_HMAC_SHA256;
 605   i->icv_size = 12;
 606
 607   i = &im->integ_algs[IPSEC_INTEG_ALG_SHA_256_128];
 608   i->op_id = VNET_CRYPTO_OP_SHA256_HMAC;
 609   i->alg = VNET_CRYPTO_ALG_HMAC_SHA256;
 610   i->icv_size = 16;
 611
 612   i = &im->integ_algs[IPSEC_INTEG_ALG_SHA_384_192];
 613   i->op_id = VNET_CRYPTO_OP_SHA384_HMAC;
 614   i->alg = VNET_CRYPTO_ALG_HMAC_SHA384;
 615   i->icv_size = 24;
 616
 617   i = &im->integ_algs[IPSEC_INTEG_ALG_SHA_512_256];
 618   i->op_id = VNET_CRYPTO_OP_SHA512_HMAC;
 619   i->alg = VNET_CRYPTO_ALG_HMAC_SHA512;
 620   i->icv_size = 32;
 621
 622   vec_validate_aligned (im->ptd, vlib_num_workers (), CLIB_CACHE_LINE_BYTES);
 623
 624   im->async_mode = 0;
 625   crypto_engine_backend_register_post_node (vm);
 626
 627   im->ipsec4_out_spd_hash_tbl = NULL;
 628   im->output_flow_cache_flag = 0;
 629   im->ipsec4_out_spd_flow_cache_entries = 0;
 630   im->epoch_count = 0;
 631   im->ipsec4_out_spd_hash_num_buckets =
 632     IPSEC4_OUT_SPD_DEFAULT_HASH_NUM_BUCKETS;
 633
 634   im->ipsec4_in_spd_hash_tbl = NULL;
 635   im->input_flow_cache_flag = 0;
 636   im->ipsec4_in_spd_flow_cache_entries = 0;
 637   im->input_epoch_count = 0;
 638   im->ipsec4_in_spd_hash_num_buckets = IPSEC4_SPD_DEFAULT_HASH_NUM_BUCKETS;
 639
 640   vec_validate_init_empty_aligned (im->next_header_registrations, 255, ~0,
 641                                    CLIB_CACHE_LINE_BYTES);
 642
 643   im->fp_spd_ipv4_out_is_enabled = 0;
 644   im->fp_spd_ipv6_out_is_enabled = 0;
 645   im->fp_spd_ipv4_in_is_enabled = 0;
 646   im->fp_spd_ipv6_in_is_enabled = 0;
 647
 648   im->fp_lookup_hash_buckets = IPSEC_FP_HASH_LOOKUP_HASH_BUCKETS;
 649
 650   return 0;
 651 }
 652
 653 VLIB_INIT_FUNCTION (ipsec_init);
 654
 655 static clib_error_t *
 656 ipsec_config (vlib_main_t *vm, unformat_input_t *input)
 657 {
 658   ipsec_main_t *im = &ipsec_main;
 659   unformat_input_t sub_input;
 660
 661   u32 ipsec4_out_spd_hash_num_buckets;
 662   u32 ipsec4_in_spd_hash_num_buckets;
 663   u32 ipsec_spd_fp_num_buckets;
 664   bool fp_spd_ip4_enabled = false;
 665   bool fp_spd_ip6_enabled = false;
 666
 667   while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
 668     {
 669       if (unformat (input, "ipv6-outbound-spd-fast-path on"))
 670         {
 671           im->fp_spd_ipv6_out_is_enabled = 1;
 672           fp_spd_ip6_enabled = true;
 673         }
 674       else if (unformat (input, "ipv6-outbound-spd-fast-path off"))
 675         im->fp_spd_ipv6_out_is_enabled = 0;
 676       else if (unformat (input, "ipv4-outbound-spd-fast-path on"))
 677         {
 678           im->fp_spd_ipv4_out_is_enabled = 1;
 679           im->output_flow_cache_flag = 0;
 680           fp_spd_ip4_enabled = true;
 681         }
 682       else if (unformat (input, "ipv4-outbound-spd-fast-path off"))
 683         im->fp_spd_ipv4_out_is_enabled = 0;
 684       else if (unformat (input, "ipv6-inbound-spd-fast-path on"))
 685         {
 686           im->fp_spd_ipv6_in_is_enabled = 1;
 687           fp_spd_ip6_enabled = true;
 688         }
 689       else if (unformat (input, "ipv6-inbound-spd-fast-path off"))
 690         im->fp_spd_ipv6_in_is_enabled = 0;
 691       else if (unformat (input, "ipv4-inbound-spd-fast-path on"))
 692         {
 693           im->fp_spd_ipv4_in_is_enabled = 1;
 694           im->input_flow_cache_flag = 0;
 695           fp_spd_ip4_enabled = true;
 696         }
 697       else if (unformat (input, "ipv4-inbound-spd-fast-path off"))
 698         im->fp_spd_ipv4_in_is_enabled = 0;
 699       else if (unformat (input, "spd-fast-path-num-buckets %d",
 700                          &ipsec_spd_fp_num_buckets))
 701         {
 702           /* Number of bihash buckets is power of 2 >= input */
 703           im->fp_lookup_hash_buckets = 1ULL
 704                                        << max_log2 (ipsec_spd_fp_num_buckets);
 705         }
 706       else if (unformat (input, "ipv4-outbound-spd-flow-cache on"))
 707         im->output_flow_cache_flag = im->fp_spd_ipv4_out_is_enabled ? 0 : 1;
 708       else if (unformat (input, "ipv4-outbound-spd-flow-cache off"))
 709         im->output_flow_cache_flag = 0;
 710       else if (unformat (input, "ipv4-outbound-spd-hash-buckets %d",
 711                          &ipsec4_out_spd_hash_num_buckets))
 712         {
 713           /* Size of hash is power of 2 >= number of buckets */
 714           im->ipsec4_out_spd_hash_num_buckets =
 715             1ULL << max_log2 (ipsec4_out_spd_hash_num_buckets);
 716         }
 717       else if (unformat (input, "ipv4-inbound-spd-flow-cache on"))
 718         im->input_flow_cache_flag = im->fp_spd_ipv4_in_is_enabled ? 0 : 1;
 719       else if (unformat (input, "ipv4-inbound-spd-flow-cache off"))
 720         im->input_flow_cache_flag = 0;
 721       else if (unformat (input, "ipv4-inbound-spd-hash-buckets %d",
 722                          &ipsec4_in_spd_hash_num_buckets))
 723         {
 724           im->ipsec4_in_spd_hash_num_buckets =
 725             1ULL << max_log2 (ipsec4_in_spd_hash_num_buckets);
 726         }
 727       else if (unformat (input, "ip4 %U", unformat_vlib_cli_sub_input,
 728                          &sub_input))
 729         {
 730           uword table_size = ~0;
 731           u32 n_buckets = ~0;
 732
 733           while (unformat_check_input (&sub_input) != UNFORMAT_END_OF_INPUT)
 734             {
 735               if (unformat (&sub_input, "num-buckets %u", &n_buckets))
 736                 ;
 737               else
 738                 return clib_error_return (0, "unknown input `%U'",
 739                                           format_unformat_error, &sub_input);
 740             }
 741
 742           ipsec_tun_table_init (AF_IP4, table_size, n_buckets);
 743         }
 744       else if (unformat (input, "ip6 %U", unformat_vlib_cli_sub_input,
 745                          &sub_input))
 746         {
 747           uword table_size = ~0;
 748           u32 n_buckets = ~0;
 749
 750           while (unformat_check_input (&sub_input) != UNFORMAT_END_OF_INPUT)
 751             {
 752               if (unformat (&sub_input, "num-buckets %u", &n_buckets))
 753                 ;
 754               else
 755                 return clib_error_return (0, "unknown input `%U'",
 756                                           format_unformat_error, &sub_input);
 757             }
 758
 759           ipsec_tun_table_init (AF_IP6, table_size, n_buckets);
 760         }
 761       else
 762         return clib_error_return (0, "unknown input `%U'",
 763                                   format_unformat_error, input);
 764     }
 765   if (im->output_flow_cache_flag)
 766     {
 767       vec_add2 (im->ipsec4_out_spd_hash_tbl, im->ipsec4_out_spd_hash_tbl,
 768                 im->ipsec4_out_spd_hash_num_buckets);
 769     }
 770   if (im->input_flow_cache_flag)
 771     {
 772       vec_add2 (im->ipsec4_in_spd_hash_tbl, im->ipsec4_in_spd_hash_tbl,
 773                 im->ipsec4_in_spd_hash_num_buckets);
 774     }
 775
 776   if (fp_spd_ip4_enabled)
 777     pool_alloc_aligned (im->fp_ip4_lookup_hashes_pool,
 778                         IPSEC_FP_IP4_HASHES_POOL_SIZE, CLIB_CACHE_LINE_BYTES);
 779
 780   if (fp_spd_ip6_enabled)
 781     pool_alloc_aligned (im->fp_ip6_lookup_hashes_pool,
 782                         IPSEC_FP_IP6_HASHES_POOL_SIZE, CLIB_CACHE_LINE_BYTES);
 783
 784   return 0;
 785 }
 786
 787 VLIB_CONFIG_FUNCTION (ipsec_config, "ipsec");
 788
 789 /*
 790  * fd.io coding-style-patch-verification: ON
 791  *
 792  * Local Variables:
 793  * eval: (c-set-style "gnu")
 794  * End:
 795  */