src/plugins/crypto_ipsecmb/ipsecmb.c

   1 /*
   2  * ipsecmb.c - Intel IPSec Multi-buffer library Crypto Engine
   3  *
   4  * Copyright (c) 2019 Cisco Systemss
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at:
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17
  18 #include <fcntl.h>
  19
  20 #include <intel-ipsec-mb.h>
  21
  22 #include <vnet/vnet.h>
  23 #include <vnet/plugin/plugin.h>
  24 #include <vpp/app/version.h>
  25 #include <vnet/crypto/crypto.h>
  26 #include <vppinfra/cpu.h>
  27
  28 typedef struct
  29 {
  30   MB_MGR *mgr;
  31   __m128i cbc_iv;
  32 } ipsecmb_per_thread_data_t;
  33
  34 typedef struct ipsecmb_main_t_
  35 {
  36   ipsecmb_per_thread_data_t *per_thread_data;
  37 } ipsecmb_main_t;
  38
  39 static ipsecmb_main_t ipsecmb_main;
  40
  41 #define foreach_ipsecmb_hmac_op                                \
  42   _(SHA1, SHA1, sha1)                                          \
  43   _(SHA256, SHA_256, sha256)                                   \
  44   _(SHA384, SHA_384, sha384)                                   \
  45   _(SHA512, SHA_512, sha512)
  46
  47 #define foreach_ipsecmb_cipher_op                              \
  48   _(AES_128_CBC, 128, 16, 16)                                  \
  49   _(AES_192_CBC, 192, 24, 16)                                  \
  50   _(AES_256_CBC, 256, 32, 16)
  51
  52 always_inline void
  53 hash_expand_keys (const MB_MGR * mgr,
  54                   const u8 * key,
  55                   u32 length,
  56                   u8 block_size,
  57                   u8 ipad[256], u8 opad[256], hash_one_block_t fn)
  58 {
  59   u8 buf[block_size];
  60   int i = 0;
  61
  62   if (length > block_size)
  63     {
  64       return;
  65     }
  66
  67   memset (buf, 0x36, sizeof (buf));
  68   for (i = 0; i < length; i++)
  69     {
  70       buf[i] ^= key[i];
  71     }
  72   fn (buf, ipad);
  73
  74   memset (buf, 0x5c, sizeof (buf));
  75
  76   for (i = 0; i < length; i++)
  77     {
  78       buf[i] ^= key[i];
  79     }
  80   fn (buf, opad);
  81 }
  82
  83 always_inline void
  84 ipsecmb_retire_hmac_job (JOB_AES_HMAC * job, u32 * n_fail)
  85 {
  86   vnet_crypto_op_t *op = job->user_data;
  87
  88   if (STS_COMPLETED != job->status)
  89     {
  90       op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
  91       *n_fail = *n_fail + 1;
  92     }
  93   else
  94     op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
  95
  96   if (op->flags & VNET_CRYPTO_OP_FLAG_HMAC_CHECK)
  97     {
  98       if ((memcmp (op->digest, job->auth_tag_output, op->digest_len)))
  99         {
 100           *n_fail = *n_fail + 1;
 101           op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
 102         }
 103     }
 104   else
 105     clib_memcpy_fast (op->digest, job->auth_tag_output, op->digest_len);
 106 }
 107
 108 static_always_inline u32
 109 ipsecmb_ops_hmac_inline (vlib_main_t * vm,
 110                          const ipsecmb_per_thread_data_t * ptd,
 111                          vnet_crypto_op_t * ops[],
 112                          u32 n_ops,
 113                          u32 block_size,
 114                          hash_one_block_t fn, JOB_HASH_ALG alg)
 115 {
 116   JOB_AES_HMAC *job;
 117   u32 i, n_fail = 0;
 118   u8 scratch[n_ops][64];
 119
 120   /*
 121    * queue all the jobs first ...
 122    */
 123   for (i = 0; i < n_ops; i++)
 124     {
 125       vnet_crypto_op_t *op = ops[i];
 126       u8 ipad[256], opad[256];
 127
 128       hash_expand_keys (ptd->mgr, op->key, op->key_len,
 129                         block_size, ipad, opad, fn);
 130
 131       job = IMB_GET_NEXT_JOB (ptd->mgr);
 132
 133       job->src = op->src;
 134       job->hash_start_src_offset_in_bytes = 0;
 135       job->msg_len_to_hash_in_bytes = op->len;
 136       job->hash_alg = alg;
 137       job->auth_tag_output_len_in_bytes = op->digest_len;
 138       job->auth_tag_output = scratch[i];
 139
 140       job->cipher_mode = NULL_CIPHER;
 141       job->cipher_direction = DECRYPT;
 142       job->chain_order = HASH_CIPHER;
 143
 144       job->aes_key_len_in_bytes = op->key_len;
 145
 146       job->u.HMAC._hashed_auth_key_xor_ipad = ipad;
 147       job->u.HMAC._hashed_auth_key_xor_opad = opad;
 148       job->user_data = op;
 149
 150       job = IMB_SUBMIT_JOB (ptd->mgr);
 151
 152       if (job)
 153         ipsecmb_retire_hmac_job (job, &n_fail);
 154     }
 155
 156   /*
 157    * .. then flush (i.e. complete) them
 158    *  We will have queued enough to satisfy the 'multi' buffer
 159    */
 160   while ((job = IMB_FLUSH_JOB (ptd->mgr)))
 161     {
 162       ipsecmb_retire_hmac_job (job, &n_fail);
 163     }
 164
 165   return n_ops - n_fail;
 166 }
 167
 168 #define _(a, b, c)                                                      \
 169 static_always_inline u32                                                \
 170 ipsecmb_ops_hmac_##a (vlib_main_t * vm,                                 \
 171                       vnet_crypto_op_t * ops[],                         \
 172                       u32 n_ops)                                        \
 173 {                                                                       \
 174   ipsecmb_per_thread_data_t *ptd;                                       \
 175   ipsecmb_main_t *imbm;                                                 \
 176                                                                         \
 177   imbm = &ipsecmb_main;                                                 \
 178   ptd = vec_elt_at_index (imbm->per_thread_data, vm->thread_index);     \
 179                                                                         \
 180   return ipsecmb_ops_hmac_inline (vm, ptd, ops, n_ops,                  \
 181                                   b##_BLOCK_SIZE,                       \
 182                                   ptd->mgr->c##_one_block,              \
 183                                   b);                                   \
 184   }
 185 foreach_ipsecmb_hmac_op;
 186 #undef _
 187
 188 #define EXPANDED_KEY_N_BYTES (16 * 15)
 189
 190 always_inline void
 191 ipsecmb_retire_cipher_job (JOB_AES_HMAC * job, u32 * n_fail)
 192 {
 193   vnet_crypto_op_t *op = job->user_data;
 194
 195   if (STS_COMPLETED != job->status)
 196     {
 197       op->status = VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
 198       *n_fail = *n_fail + 1;
 199     }
 200   else
 201     op->status = VNET_CRYPTO_OP_STATUS_COMPLETED;
 202 }
 203
 204 static_always_inline u32
 205 ipsecmb_ops_cipher_inline (vlib_main_t * vm,
 206                            ipsecmb_per_thread_data_t * ptd,
 207                            vnet_crypto_op_t * ops[],
 208                            u32 n_ops, u32 key_len, u32 iv_len,
 209                            keyexp_t fn, JOB_CIPHER_DIRECTION direction)
 210 {
 211   JOB_AES_HMAC *job;
 212   u32 i, n_fail = 0;
 213
 214   /*
 215    * queue all the jobs first ...
 216    */
 217   for (i = 0; i < n_ops; i++)
 218     {
 219       u8 aes_enc_key_expanded[EXPANDED_KEY_N_BYTES];
 220       u8 aes_dec_key_expanded[EXPANDED_KEY_N_BYTES];
 221       vnet_crypto_op_t *op = ops[i];
 222       __m128i iv;
 223
 224       fn (op->key, aes_enc_key_expanded, aes_dec_key_expanded);
 225
 226       job = IMB_GET_NEXT_JOB (ptd->mgr);
 227
 228       job->src = op->src;
 229       job->dst = op->dst;
 230       job->msg_len_to_cipher_in_bytes = op->len;
 231       job->cipher_start_src_offset_in_bytes = 0;
 232
 233       job->hash_alg = NULL_HASH;
 234       job->cipher_mode = CBC;
 235       job->cipher_direction = direction;
 236       job->chain_order = (direction == ENCRYPT ? CIPHER_HASH : HASH_CIPHER);
 237
 238       if ((direction == ENCRYPT) && (op->flags & VNET_CRYPTO_OP_FLAG_INIT_IV))
 239         {
 240           iv = ptd->cbc_iv;
 241           _mm_storeu_si128 ((__m128i *) op->iv, iv);
 242           ptd->cbc_iv = _mm_aesenc_si128 (iv, iv);
 243         }
 244
 245       job->aes_key_len_in_bytes = key_len;
 246       job->aes_enc_key_expanded = aes_enc_key_expanded;
 247       job->aes_dec_key_expanded = aes_dec_key_expanded;
 248       job->iv = op->iv;
 249       job->iv_len_in_bytes = iv_len;
 250
 251       job->user_data = op;
 252
 253       job = IMB_SUBMIT_JOB (ptd->mgr);
 254
 255       if (job)
 256         ipsecmb_retire_cipher_job (job, &n_fail);
 257     }
 258
 259   /*
 260    * .. then flush (i.e. complete) them
 261    *  We will have queued enough to satisfy the 'multi' buffer
 262    */
 263   while ((job = IMB_FLUSH_JOB (ptd->mgr)))
 264     {
 265       ipsecmb_retire_cipher_job (job, &n_fail);
 266     }
 267
 268   return n_ops - n_fail;
 269 }
 270
 271 #define _(a, b, c, d)                                                   \
 272 static_always_inline u32                                                \
 273 ipsecmb_ops_cipher_enc_##a (vlib_main_t * vm,                           \
 274                             vnet_crypto_op_t * ops[],                   \
 275                             u32 n_ops)                                  \
 276 {                                                                       \
 277   ipsecmb_per_thread_data_t *ptd;                                       \
 278   ipsecmb_main_t *imbm;                                                 \
 279                                                                         \
 280   imbm = &ipsecmb_main;                                                 \
 281   ptd = vec_elt_at_index (imbm->per_thread_data, vm->thread_index);     \
 282                                                                         \
 283   return ipsecmb_ops_cipher_inline (vm, ptd, ops, n_ops, c, d,          \
 284                                     ptd->mgr->keyexp_##b,               \
 285                                     ENCRYPT);                           \
 286   }
 287 foreach_ipsecmb_cipher_op;
 288 #undef _
 289
 290 #define _(a, b, c, d)                                                   \
 291 static_always_inline u32                                                \
 292 ipsecmb_ops_cipher_dec_##a (vlib_main_t * vm,                           \
 293                             vnet_crypto_op_t * ops[],                   \
 294                             u32 n_ops)                                  \
 295 {                                                                       \
 296   ipsecmb_per_thread_data_t *ptd;                                       \
 297   ipsecmb_main_t *imbm;                                                 \
 298                                                                         \
 299   imbm = &ipsecmb_main;                                                 \
 300   ptd = vec_elt_at_index (imbm->per_thread_data, vm->thread_index);     \
 301                                                                         \
 302   return ipsecmb_ops_cipher_inline (vm, ptd, ops, n_ops, c, d,          \
 303                                     ptd->mgr->keyexp_##b,               \
 304                                     DECRYPT);                           \
 305   }
 306 foreach_ipsecmb_cipher_op;
 307 #undef _
 308
 309 clib_error_t *
 310 crypto_ipsecmb_iv_init (ipsecmb_main_t * imbm)
 311 {
 312   ipsecmb_per_thread_data_t *ptd;
 313   clib_error_t *err = 0;
 314   int fd;
 315
 316   if ((fd = open ("/dev/urandom", O_RDONLY)) < 0)
 317     return clib_error_return_unix (0, "failed to open '/dev/urandom'");
 318
 319   vec_foreach (ptd, imbm->per_thread_data)
 320   {
 321     if (read (fd, &ptd->cbc_iv, sizeof (ptd->cbc_iv)) != sizeof (ptd->cbc_iv))
 322       {
 323         err = clib_error_return_unix (0, "'/dev/urandom' read failure");
 324         close (fd);
 325         return (err);
 326       }
 327   }
 328
 329   close (fd);
 330   return (NULL);
 331 }
 332
 333 static clib_error_t *
 334 crypto_ipsecmb_init (vlib_main_t * vm)
 335 {
 336   ipsecmb_main_t *imbm = &ipsecmb_main;
 337   ipsecmb_per_thread_data_t *ptd;
 338   vlib_thread_main_t *tm = vlib_get_thread_main ();
 339   clib_error_t *error;
 340   u32 eidx;
 341
 342   if ((error = vlib_call_init_function (vm, vnet_crypto_init)))
 343     return error;
 344
 345   /*
 346    * A priority that is better than OpenSSL but worse than VPP natvie
 347    */
 348   eidx = vnet_crypto_register_engine (vm, "ipsecmb", 80,
 349                                       "Intel IPSEC multi-buffer");
 350
 351   vec_validate (imbm->per_thread_data, tm->n_vlib_mains - 1);
 352
 353   if (clib_cpu_supports_avx512f ())
 354     {
 355       vec_foreach (ptd, imbm->per_thread_data)
 356       {
 357         ptd->mgr = alloc_mb_mgr (0);
 358         init_mb_mgr_avx512 (ptd->mgr);
 359       }
 360     }
 361   else if (clib_cpu_supports_avx2 ())
 362     {
 363       vec_foreach (ptd, imbm->per_thread_data)
 364       {
 365         ptd->mgr = alloc_mb_mgr (0);
 366         init_mb_mgr_avx2 (ptd->mgr);
 367       }
 368     }
 369   else
 370     {
 371       vec_foreach (ptd, imbm->per_thread_data)
 372       {
 373         ptd->mgr = alloc_mb_mgr (0);
 374         init_mb_mgr_sse (ptd->mgr);
 375       }
 376     }
 377
 378   if (clib_cpu_supports_x86_aes () && (error = crypto_ipsecmb_iv_init (imbm)))
 379     return (error);
 380
 381
 382 #define _(a, b, c)                                                       \
 383   vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_HMAC, \
 384                                     ipsecmb_ops_hmac_##a);               \
 385
 386   foreach_ipsecmb_hmac_op;
 387 #undef _
 388 #define _(a, b, c, d)                                                   \
 389   vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_ENC, \
 390                                     ipsecmb_ops_cipher_enc_##a);        \
 391
 392   foreach_ipsecmb_cipher_op;
 393 #undef _
 394 #define _(a, b, c, d)                                                   \
 395   vnet_crypto_register_ops_handler (vm, eidx, VNET_CRYPTO_OP_##a##_DEC, \
 396                                     ipsecmb_ops_cipher_dec_##a);        \
 397
 398   foreach_ipsecmb_cipher_op;
 399 #undef _
 400
 401   return (NULL);
 402 }
 403
 404 VLIB_INIT_FUNCTION (crypto_ipsecmb_init);
 405
 406 /* *INDENT-OFF* */
 407 VLIB_PLUGIN_REGISTER () =
 408 {
 409   .version = VPP_BUILD_VER,
 410   .description = "Intel IPSEC multi-buffer",
 411 };
 412 /* *INDENT-ON* */
 413
 414 /*
 415  * fd.io coding-style-patch-verification: ON
 416  *
 417  * Local Variables:
 418  * eval: (c-set-style "gnu")
 419  * End:
 420  */