drivers/crypto/openssl/rte_openssl_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016-2017 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_common.h>
  34 #include <rte_hexdump.h>
  35 #include <rte_cryptodev.h>
  36 #include <rte_cryptodev_pmd.h>
  37 #include <rte_bus_vdev.h>
  38 #include <rte_malloc.h>
  39 #include <rte_cpuflags.h>
  40
  41 #include <openssl/hmac.h>
  42 #include <openssl/evp.h>
  43
  44 #include "rte_openssl_pmd_private.h"
  45
  46 #define DES_BLOCK_SIZE 8
  47
  48 static uint8_t cryptodev_driver_id;
  49
  50 #if (OPENSSL_VERSION_NUMBER < 0x10100000L)
  51 static HMAC_CTX *HMAC_CTX_new(void)
  52 {
  53         HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
  54
  55         if (ctx != NULL)
  56                 HMAC_CTX_init(ctx);
  57         return ctx;
  58 }
  59
  60 static void HMAC_CTX_free(HMAC_CTX *ctx)
  61 {
  62         if (ctx != NULL) {
  63                 HMAC_CTX_cleanup(ctx);
  64                 OPENSSL_free(ctx);
  65         }
  66 }
  67 #endif
  68
  69 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  70
  71 /*----------------------------------------------------------------------------*/
  72
  73 /**
  74  * Increment counter by 1
  75  * Counter is 64 bit array, big-endian
  76  */
  77 static void
  78 ctr_inc(uint8_t *ctr)
  79 {
  80         uint64_t *ctr64 = (uint64_t *)ctr;
  81
  82         *ctr64 = __builtin_bswap64(*ctr64);
  83         (*ctr64)++;
  84         *ctr64 = __builtin_bswap64(*ctr64);
  85 }
  86
  87 /*
  88  *------------------------------------------------------------------------------
  89  * Session Prepare
  90  *------------------------------------------------------------------------------
  91  */
  92
  93 /** Get xform chain order */
  94 static enum openssl_chain_order
  95 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  96 {
  97         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  98
  99         if (xform != NULL) {
 100                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 101                         if (xform->next == NULL)
 102                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
 103                         else if (xform->next->type ==
 104                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
 105                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
 106                 }
 107                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 108                         if (xform->next == NULL)
 109                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
 110                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 111                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
 112                 }
 113                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD)
 114                         res = OPENSSL_CHAIN_COMBINED;
 115         }
 116
 117         return res;
 118 }
 119
 120 /** Get session cipher key from input cipher key */
 121 static void
 122 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
 123 {
 124         memcpy(session_key, input_key, keylen);
 125 }
 126
 127 /** Get key ede 24 bytes standard from input key */
 128 static int
 129 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 130 {
 131         int res = 0;
 132
 133         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 134         switch (keylen) {
 135         case 24:
 136                 memcpy(key_ede, key, 24);
 137                 break;
 138         case 16:
 139                 /* K3 = K1 */
 140                 memcpy(key_ede, key, 16);
 141                 memcpy(key_ede + 16, key, 8);
 142                 break;
 143         case 8:
 144                 /* K1 = K2 = K3 (DES compatibility) */
 145                 memcpy(key_ede, key, 8);
 146                 memcpy(key_ede + 8, key, 8);
 147                 memcpy(key_ede + 16, key, 8);
 148                 break;
 149         default:
 150                 OPENSSL_LOG_ERR("Unsupported key size");
 151                 res = -EINVAL;
 152         }
 153
 154         return res;
 155 }
 156
 157 /** Get adequate openssl function for input cipher algorithm */
 158 static uint8_t
 159 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 160                 const EVP_CIPHER **algo)
 161 {
 162         int res = 0;
 163
 164         if (algo != NULL) {
 165                 switch (sess_algo) {
 166                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 167                         switch (keylen) {
 168                         case 16:
 169                                 *algo = EVP_des_ede_cbc();
 170                                 break;
 171                         case 24:
 172                                 *algo = EVP_des_ede3_cbc();
 173                                 break;
 174                         default:
 175                                 res = -EINVAL;
 176                         }
 177                         break;
 178                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 179                         break;
 180                 case RTE_CRYPTO_CIPHER_AES_CBC:
 181                         switch (keylen) {
 182                         case 16:
 183                                 *algo = EVP_aes_128_cbc();
 184                                 break;
 185                         case 24:
 186                                 *algo = EVP_aes_192_cbc();
 187                                 break;
 188                         case 32:
 189                                 *algo = EVP_aes_256_cbc();
 190                                 break;
 191                         default:
 192                                 res = -EINVAL;
 193                         }
 194                         break;
 195                 case RTE_CRYPTO_CIPHER_AES_CTR:
 196                         switch (keylen) {
 197                         case 16:
 198                                 *algo = EVP_aes_128_ctr();
 199                                 break;
 200                         case 24:
 201                                 *algo = EVP_aes_192_ctr();
 202                                 break;
 203                         case 32:
 204                                 *algo = EVP_aes_256_ctr();
 205                                 break;
 206                         default:
 207                                 res = -EINVAL;
 208                         }
 209                         break;
 210                 default:
 211                         res = -EINVAL;
 212                         break;
 213                 }
 214         } else {
 215                 res = -EINVAL;
 216         }
 217
 218         return res;
 219 }
 220
 221 /** Get adequate openssl function for input auth algorithm */
 222 static uint8_t
 223 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 224                 const EVP_MD **algo)
 225 {
 226         int res = 0;
 227
 228         if (algo != NULL) {
 229                 switch (sessalgo) {
 230                 case RTE_CRYPTO_AUTH_MD5:
 231                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 232                         *algo = EVP_md5();
 233                         break;
 234                 case RTE_CRYPTO_AUTH_SHA1:
 235                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 236                         *algo = EVP_sha1();
 237                         break;
 238                 case RTE_CRYPTO_AUTH_SHA224:
 239                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 240                         *algo = EVP_sha224();
 241                         break;
 242                 case RTE_CRYPTO_AUTH_SHA256:
 243                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 244                         *algo = EVP_sha256();
 245                         break;
 246                 case RTE_CRYPTO_AUTH_SHA384:
 247                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 248                         *algo = EVP_sha384();
 249                         break;
 250                 case RTE_CRYPTO_AUTH_SHA512:
 251                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 252                         *algo = EVP_sha512();
 253                         break;
 254                 default:
 255                         res = -EINVAL;
 256                         break;
 257                 }
 258         } else {
 259                 res = -EINVAL;
 260         }
 261
 262         return res;
 263 }
 264
 265 /** Get adequate openssl function for input cipher algorithm */
 266 static uint8_t
 267 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen,
 268                 const EVP_CIPHER **algo)
 269 {
 270         int res = 0;
 271
 272         if (algo != NULL) {
 273                 switch (sess_algo) {
 274                 case RTE_CRYPTO_AEAD_AES_GCM:
 275                         switch (keylen) {
 276                         case 16:
 277                                 *algo = EVP_aes_128_gcm();
 278                                 break;
 279                         case 24:
 280                                 *algo = EVP_aes_192_gcm();
 281                                 break;
 282                         case 32:
 283                                 *algo = EVP_aes_256_gcm();
 284                                 break;
 285                         default:
 286                                 res = -EINVAL;
 287                         }
 288                         break;
 289                 case RTE_CRYPTO_AEAD_AES_CCM:
 290                         switch (keylen) {
 291                         case 16:
 292                                 *algo = EVP_aes_128_ccm();
 293                                 break;
 294                         case 24:
 295                                 *algo = EVP_aes_192_ccm();
 296                                 break;
 297                         case 32:
 298                                 *algo = EVP_aes_256_ccm();
 299                                 break;
 300                         default:
 301                                 res = -EINVAL;
 302                         }
 303                         break;
 304                 default:
 305                         res = -EINVAL;
 306                         break;
 307                 }
 308         } else {
 309                 res = -EINVAL;
 310         }
 311
 312         return res;
 313 }
 314
 315 /* Set session AEAD encryption parameters */
 316 static int
 317 openssl_set_sess_aead_enc_param(struct openssl_session *sess,
 318                 enum rte_crypto_aead_algorithm algo,
 319                 uint8_t tag_len, uint8_t *key)
 320 {
 321         int iv_type = 0;
 322         unsigned int do_ccm;
 323
 324         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT;
 325         sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE;
 326
 327         /* Select AEAD algo */
 328         switch (algo) {
 329         case RTE_CRYPTO_AEAD_AES_GCM:
 330                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 331                 if (tag_len != 16)
 332                         return -EINVAL;
 333                 do_ccm = 0;
 334                 break;
 335         case RTE_CRYPTO_AEAD_AES_CCM:
 336                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 337                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 338                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 339                         return -EINVAL;
 340                 do_ccm = 1;
 341                 break;
 342         default:
 343                 return -ENOTSUP;
 344         }
 345
 346         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 347         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 348
 349         if (get_aead_algo(algo, sess->cipher.key.length,
 350                         &sess->cipher.evp_algo) != 0)
 351                 return -EINVAL;
 352
 353         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 354
 355         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 356
 357         if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 358                         NULL, NULL, NULL) <= 0)
 359                 return -EINVAL;
 360
 361         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length,
 362                         NULL) <= 0)
 363                 return -EINVAL;
 364
 365         if (do_ccm)
 366                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 367                                 tag_len, NULL);
 368
 369         if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 370                 return -EINVAL;
 371
 372         return 0;
 373 }
 374
 375 /* Set session AEAD decryption parameters */
 376 static int
 377 openssl_set_sess_aead_dec_param(struct openssl_session *sess,
 378                 enum rte_crypto_aead_algorithm algo,
 379                 uint8_t tag_len, uint8_t *key)
 380 {
 381         int iv_type = 0;
 382         unsigned int do_ccm = 0;
 383
 384         sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT;
 385         sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY;
 386
 387         /* Select AEAD algo */
 388         switch (algo) {
 389         case RTE_CRYPTO_AEAD_AES_GCM:
 390                 iv_type = EVP_CTRL_GCM_SET_IVLEN;
 391                 if (tag_len != 16)
 392                         return -EINVAL;
 393                 break;
 394         case RTE_CRYPTO_AEAD_AES_CCM:
 395                 iv_type = EVP_CTRL_CCM_SET_IVLEN;
 396                 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */
 397                 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1)
 398                         return -EINVAL;
 399                 do_ccm = 1;
 400                 break;
 401         default:
 402                 return -ENOTSUP;
 403         }
 404
 405         sess->cipher.mode = OPENSSL_CIPHER_LIB;
 406         sess->cipher.ctx = EVP_CIPHER_CTX_new();
 407
 408         if (get_aead_algo(algo, sess->cipher.key.length,
 409                         &sess->cipher.evp_algo) != 0)
 410                 return -EINVAL;
 411
 412         get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data);
 413
 414         sess->chain_order = OPENSSL_CHAIN_COMBINED;
 415
 416         if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo,
 417                         NULL, NULL, NULL) <= 0)
 418                 return -EINVAL;
 419
 420         if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type,
 421                         sess->iv.length, NULL) <= 0)
 422                 return -EINVAL;
 423
 424         if (do_ccm)
 425                 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG,
 426                                 tag_len, NULL);
 427
 428         if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0)
 429                 return -EINVAL;
 430
 431         return 0;
 432 }
 433
 434 /** Set session cipher parameters */
 435 static int
 436 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 437                 const struct rte_crypto_sym_xform *xform)
 438 {
 439         /* Select cipher direction */
 440         sess->cipher.direction = xform->cipher.op;
 441         /* Select cipher key */
 442         sess->cipher.key.length = xform->cipher.key.length;
 443
 444         /* Set IV parameters */
 445         sess->iv.offset = xform->cipher.iv.offset;
 446         sess->iv.length = xform->cipher.iv.length;
 447
 448         /* Select cipher algo */
 449         switch (xform->cipher.algo) {
 450         case RTE_CRYPTO_CIPHER_3DES_CBC:
 451         case RTE_CRYPTO_CIPHER_AES_CBC:
 452         case RTE_CRYPTO_CIPHER_AES_CTR:
 453                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 454                 sess->cipher.algo = xform->cipher.algo;
 455                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 456
 457                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 458                                 &sess->cipher.evp_algo) != 0)
 459                         return -EINVAL;
 460
 461                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 462                         sess->cipher.key.data);
 463                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 464                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 465                                         sess->cipher.evp_algo,
 466                                         NULL, xform->cipher.key.data,
 467                                         NULL) != 1) {
 468                                 return -EINVAL;
 469                         }
 470                 } else if (sess->cipher.direction ==
 471                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 472                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 473                                         sess->cipher.evp_algo,
 474                                         NULL, xform->cipher.key.data,
 475                                         NULL) != 1) {
 476                                 return -EINVAL;
 477                         }
 478                 }
 479
 480                 break;
 481
 482         case RTE_CRYPTO_CIPHER_3DES_CTR:
 483                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 484                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 485
 486                 if (get_cipher_key_ede(xform->cipher.key.data,
 487                                 sess->cipher.key.length,
 488                                 sess->cipher.key.data) != 0)
 489                         return -EINVAL;
 490                 break;
 491
 492         case RTE_CRYPTO_CIPHER_DES_CBC:
 493                 sess->cipher.algo = xform->cipher.algo;
 494                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 495                 sess->cipher.evp_algo = EVP_des_cbc();
 496
 497                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 498                         sess->cipher.key.data);
 499                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 500                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 501                                         sess->cipher.evp_algo,
 502                                         NULL, xform->cipher.key.data,
 503                                         NULL) != 1) {
 504                                 return -EINVAL;
 505                         }
 506                 } else if (sess->cipher.direction ==
 507                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 508                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 509                                         sess->cipher.evp_algo,
 510                                         NULL, xform->cipher.key.data,
 511                                         NULL) != 1) {
 512                                 return -EINVAL;
 513                         }
 514                 }
 515
 516                 break;
 517
 518         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 519                 sess->cipher.algo = xform->cipher.algo;
 520                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 521                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 522                 sess->cipher.evp_algo = EVP_des_cbc();
 523
 524                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 525                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 526                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 527                                 NULL, xform->cipher.key.data, 0) != 1)
 528                         return -EINVAL;
 529
 530                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 531                         sess->cipher.key.data);
 532                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
 533                         if (EVP_EncryptInit_ex(sess->cipher.ctx,
 534                                         sess->cipher.evp_algo,
 535                                         NULL, xform->cipher.key.data,
 536                                         NULL) != 1) {
 537                                 return -EINVAL;
 538                         }
 539                 } else if (sess->cipher.direction ==
 540                                 RTE_CRYPTO_CIPHER_OP_DECRYPT) {
 541                         if (EVP_DecryptInit_ex(sess->cipher.ctx,
 542                                         sess->cipher.evp_algo,
 543                                         NULL, xform->cipher.key.data,
 544                                         NULL) != 1) {
 545                                 return -EINVAL;
 546                         }
 547                 }
 548
 549                 break;
 550         default:
 551                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 552                 return -ENOTSUP;
 553         }
 554
 555         return 0;
 556 }
 557
 558 /* Set session auth parameters */
 559 static int
 560 openssl_set_session_auth_parameters(struct openssl_session *sess,
 561                 const struct rte_crypto_sym_xform *xform)
 562 {
 563         /* Select auth generate/verify */
 564         sess->auth.operation = xform->auth.op;
 565         sess->auth.algo = xform->auth.algo;
 566
 567         sess->auth.digest_length = xform->auth.digest_length;
 568
 569         /* Select auth algo */
 570         switch (xform->auth.algo) {
 571         case RTE_CRYPTO_AUTH_AES_GMAC:
 572                 /*
 573                  * OpenSSL requires GMAC to be a GCM operation
 574                  * with no cipher data length
 575                  */
 576                 sess->cipher.key.length = xform->auth.key.length;
 577
 578                 /* Set IV parameters */
 579                 sess->iv.offset = xform->auth.iv.offset;
 580                 sess->iv.length = xform->auth.iv.length;
 581
 582                 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE)
 583                         return openssl_set_sess_aead_enc_param(sess,
 584                                                 RTE_CRYPTO_AEAD_AES_GCM,
 585                                                 xform->auth.digest_length,
 586                                                 xform->auth.key.data);
 587                 else
 588                         return openssl_set_sess_aead_dec_param(sess,
 589                                                 RTE_CRYPTO_AEAD_AES_GCM,
 590                                                 xform->auth.digest_length,
 591                                                 xform->auth.key.data);
 592                 break;
 593
 594         case RTE_CRYPTO_AUTH_MD5:
 595         case RTE_CRYPTO_AUTH_SHA1:
 596         case RTE_CRYPTO_AUTH_SHA224:
 597         case RTE_CRYPTO_AUTH_SHA256:
 598         case RTE_CRYPTO_AUTH_SHA384:
 599         case RTE_CRYPTO_AUTH_SHA512:
 600                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 601                 if (get_auth_algo(xform->auth.algo,
 602                                 &sess->auth.auth.evp_algo) != 0)
 603                         return -EINVAL;
 604                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 605                 break;
 606
 607         case RTE_CRYPTO_AUTH_MD5_HMAC:
 608         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 609         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 610         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 611         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 612         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 613                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 614                 sess->auth.hmac.ctx = HMAC_CTX_new();
 615                 if (get_auth_algo(xform->auth.algo,
 616                                 &sess->auth.hmac.evp_algo) != 0)
 617                         return -EINVAL;
 618
 619                 if (HMAC_Init_ex(sess->auth.hmac.ctx,
 620                                 xform->auth.key.data,
 621                                 xform->auth.key.length,
 622                                 sess->auth.hmac.evp_algo, NULL) != 1)
 623                         return -EINVAL;
 624                 break;
 625
 626         default:
 627                 return -ENOTSUP;
 628         }
 629
 630         return 0;
 631 }
 632
 633 /* Set session AEAD parameters */
 634 static int
 635 openssl_set_session_aead_parameters(struct openssl_session *sess,
 636                 const struct rte_crypto_sym_xform *xform)
 637 {
 638         /* Select cipher key */
 639         sess->cipher.key.length = xform->aead.key.length;
 640
 641         /* Set IV parameters */
 642         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM)
 643                 /*
 644                  * For AES-CCM, the actual IV is placed
 645                  * one byte after the start of the IV field,
 646                  * according to the API.
 647                  */
 648                 sess->iv.offset = xform->aead.iv.offset + 1;
 649         else
 650                 sess->iv.offset = xform->aead.iv.offset;
 651
 652         sess->iv.length = xform->aead.iv.length;
 653
 654         sess->auth.aad_length = xform->aead.aad_length;
 655         sess->auth.digest_length = xform->aead.digest_length;
 656
 657         sess->aead_algo = xform->aead.algo;
 658         /* Select cipher direction */
 659         if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 660                 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo,
 661                                 xform->aead.digest_length, xform->aead.key.data);
 662         else
 663                 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo,
 664                                 xform->aead.digest_length, xform->aead.key.data);
 665 }
 666
 667 /** Parse crypto xform chain and set private session parameters */
 668 int
 669 openssl_set_session_parameters(struct openssl_session *sess,
 670                 const struct rte_crypto_sym_xform *xform)
 671 {
 672         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 673         const struct rte_crypto_sym_xform *auth_xform = NULL;
 674         const struct rte_crypto_sym_xform *aead_xform = NULL;
 675         int ret;
 676
 677         sess->chain_order = openssl_get_chain_order(xform);
 678         switch (sess->chain_order) {
 679         case OPENSSL_CHAIN_ONLY_CIPHER:
 680                 cipher_xform = xform;
 681                 break;
 682         case OPENSSL_CHAIN_ONLY_AUTH:
 683                 auth_xform = xform;
 684                 break;
 685         case OPENSSL_CHAIN_CIPHER_AUTH:
 686                 cipher_xform = xform;
 687                 auth_xform = xform->next;
 688                 break;
 689         case OPENSSL_CHAIN_AUTH_CIPHER:
 690                 auth_xform = xform;
 691                 cipher_xform = xform->next;
 692                 break;
 693         case OPENSSL_CHAIN_COMBINED:
 694                 aead_xform = xform;
 695                 break;
 696         default:
 697                 return -EINVAL;
 698         }
 699
 700         /* Default IV length = 0 */
 701         sess->iv.length = 0;
 702
 703         /* cipher_xform must be check before auth_xform */
 704         if (cipher_xform) {
 705                 ret = openssl_set_session_cipher_parameters(
 706                                 sess, cipher_xform);
 707                 if (ret != 0) {
 708                         OPENSSL_LOG_ERR(
 709                                 "Invalid/unsupported cipher parameters");
 710                         return ret;
 711                 }
 712         }
 713
 714         if (auth_xform) {
 715                 ret = openssl_set_session_auth_parameters(sess, auth_xform);
 716                 if (ret != 0) {
 717                         OPENSSL_LOG_ERR(
 718                                 "Invalid/unsupported auth parameters");
 719                         return ret;
 720                 }
 721         }
 722
 723         if (aead_xform) {
 724                 ret = openssl_set_session_aead_parameters(sess, aead_xform);
 725                 if (ret != 0) {
 726                         OPENSSL_LOG_ERR(
 727                                 "Invalid/unsupported AEAD parameters");
 728                         return ret;
 729                 }
 730         }
 731
 732         return 0;
 733 }
 734
 735 /** Reset private session parameters */
 736 void
 737 openssl_reset_session(struct openssl_session *sess)
 738 {
 739         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 740
 741         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 742                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 743
 744         switch (sess->auth.mode) {
 745         case OPENSSL_AUTH_AS_AUTH:
 746                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 747                 break;
 748         case OPENSSL_AUTH_AS_HMAC:
 749                 EVP_PKEY_free(sess->auth.hmac.pkey);
 750                 HMAC_CTX_free(sess->auth.hmac.ctx);
 751                 break;
 752         default:
 753                 break;
 754         }
 755 }
 756
 757 /** Provide session for operation */
 758 static struct openssl_session *
 759 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 760 {
 761         struct openssl_session *sess = NULL;
 762
 763         if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) {
 764                 /* get existing session */
 765                 if (likely(op->sym->session != NULL))
 766                         sess = (struct openssl_session *)
 767                                         get_session_private_data(
 768                                         op->sym->session,
 769                                         cryptodev_driver_id);
 770         } else {
 771                 /* provide internal session */
 772                 void *_sess = NULL;
 773                 void *_sess_private_data = NULL;
 774
 775                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess))
 776                         return NULL;
 777
 778                 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data))
 779                         return NULL;
 780
 781                 sess = (struct openssl_session *)_sess_private_data;
 782
 783                 if (unlikely(openssl_set_session_parameters(sess,
 784                                 op->sym->xform) != 0)) {
 785                         rte_mempool_put(qp->sess_mp, _sess);
 786                         rte_mempool_put(qp->sess_mp, _sess_private_data);
 787                         sess = NULL;
 788                 }
 789                 op->sym->session = (struct rte_cryptodev_sym_session *)_sess;
 790                 set_session_private_data(op->sym->session, cryptodev_driver_id,
 791                         _sess_private_data);
 792         }
 793
 794         if (sess == NULL)
 795                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 796
 797         return sess;
 798 }
 799
 800 /*
 801  *------------------------------------------------------------------------------
 802  * Process Operations
 803  *------------------------------------------------------------------------------
 804  */
 805 static inline int
 806 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 807                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 808 {
 809         struct rte_mbuf *m;
 810         int dstlen;
 811         int l, n = srclen;
 812         uint8_t *src;
 813
 814         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 815                         m = m->next)
 816                 offset -= rte_pktmbuf_data_len(m);
 817
 818         if (m == 0)
 819                 return -1;
 820
 821         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 822
 823         l = rte_pktmbuf_data_len(m) - offset;
 824         if (srclen <= l) {
 825                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 826                         return -1;
 827                 *dst += l;
 828                 return 0;
 829         }
 830
 831         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 832                 return -1;
 833
 834         *dst += dstlen;
 835         n -= l;
 836
 837         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 838                 src = rte_pktmbuf_mtod(m, uint8_t *);
 839                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 840                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 841                         return -1;
 842                 *dst += dstlen;
 843                 n -= l;
 844         }
 845
 846         return 0;
 847 }
 848
 849 static inline int
 850 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 851                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 852 {
 853         struct rte_mbuf *m;
 854         int dstlen;
 855         int l, n = srclen;
 856         uint8_t *src;
 857
 858         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 859                         m = m->next)
 860                 offset -= rte_pktmbuf_data_len(m);
 861
 862         if (m == 0)
 863                 return -1;
 864
 865         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 866
 867         l = rte_pktmbuf_data_len(m) - offset;
 868         if (srclen <= l) {
 869                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 870                         return -1;
 871                 *dst += l;
 872                 return 0;
 873         }
 874
 875         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 876                 return -1;
 877
 878         *dst += dstlen;
 879         n -= l;
 880
 881         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 882                 src = rte_pktmbuf_mtod(m, uint8_t *);
 883                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 884                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 885                         return -1;
 886                 *dst += dstlen;
 887                 n -= l;
 888         }
 889
 890         return 0;
 891 }
 892
 893 /** Process standard openssl cipher encryption */
 894 static int
 895 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 896                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 897 {
 898         int totlen;
 899
 900         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 901                 goto process_cipher_encrypt_err;
 902
 903         EVP_CIPHER_CTX_set_padding(ctx, 0);
 904
 905         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 906                         srclen, ctx))
 907                 goto process_cipher_encrypt_err;
 908
 909         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 910                 goto process_cipher_encrypt_err;
 911
 912         return 0;
 913
 914 process_cipher_encrypt_err:
 915         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
 916         return -EINVAL;
 917 }
 918
 919 /** Process standard openssl cipher encryption */
 920 static int
 921 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 922                 uint8_t *iv, int srclen,
 923                 EVP_CIPHER_CTX *ctx)
 924 {
 925         uint8_t i;
 926         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 927         int encrypted_ivlen;
 928
 929         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 930                         iv, DES_BLOCK_SIZE) <= 0)
 931                 goto process_cipher_encrypt_err;
 932
 933         for (i = 0; i < srclen; i++)
 934                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 935
 936         return 0;
 937
 938 process_cipher_encrypt_err:
 939         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
 940         return -EINVAL;
 941 }
 942 /** Process standard openssl cipher decryption */
 943 static int
 944 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 945                 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx)
 946 {
 947         int totlen;
 948
 949         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
 950                 goto process_cipher_decrypt_err;
 951
 952         EVP_CIPHER_CTX_set_padding(ctx, 0);
 953
 954         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 955                         srclen, ctx))
 956                 goto process_cipher_decrypt_err;
 957
 958         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 959                 goto process_cipher_decrypt_err;
 960         return 0;
 961
 962 process_cipher_decrypt_err:
 963         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
 964         return -EINVAL;
 965 }
 966
 967 /** Process cipher des 3 ctr encryption, decryption algorithm */
 968 static int
 969 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 970                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 971                 EVP_CIPHER_CTX *ctx)
 972 {
 973         uint8_t ebuf[8], ctr[8];
 974         int unused, n;
 975         struct rte_mbuf *m;
 976         uint8_t *src;
 977         int l;
 978
 979         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 980                         m = m->next)
 981                 offset -= rte_pktmbuf_data_len(m);
 982
 983         if (m == 0)
 984                 goto process_cipher_des3ctr_err;
 985
 986         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 987         l = rte_pktmbuf_data_len(m) - offset;
 988
 989         /* We use 3DES encryption also for decryption.
 990          * IV is not important for 3DES ecb
 991          */
 992         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
 993                 goto process_cipher_des3ctr_err;
 994
 995         memcpy(ctr, iv, 8);
 996
 997         for (n = 0; n < srclen; n++) {
 998                 if (n % 8 == 0) {
 999                         if (EVP_EncryptUpdate(ctx,
1000                                         (unsigned char *)&ebuf, &unused,
1001                                         (const unsigned char *)&ctr, 8) <= 0)
1002                                 goto process_cipher_des3ctr_err;
1003                         ctr_inc(ctr);
1004                 }
1005                 dst[n] = *(src++) ^ ebuf[n % 8];
1006
1007                 l--;
1008                 if (!l) {
1009                         m = m->next;
1010                         if (m) {
1011                                 src = rte_pktmbuf_mtod(m, uint8_t *);
1012                                 l = rte_pktmbuf_data_len(m);
1013                         }
1014                 }
1015         }
1016
1017         return 0;
1018
1019 process_cipher_des3ctr_err:
1020         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
1021         return -EINVAL;
1022 }
1023
1024 /** Process AES-GCM encrypt algorithm */
1025 static int
1026 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1027                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1028                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1029 {
1030         int len = 0, unused = 0;
1031         uint8_t empty[] = {};
1032
1033         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1034                 goto process_auth_encryption_gcm_err;
1035
1036         if (aadlen > 0)
1037                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1038                         goto process_auth_encryption_gcm_err;
1039
1040         if (srclen > 0)
1041                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1042                                 srclen, ctx))
1043                         goto process_auth_encryption_gcm_err;
1044
1045         /* Workaround open ssl bug in version less then 1.0.1f */
1046         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1047                 goto process_auth_encryption_gcm_err;
1048
1049         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1050                 goto process_auth_encryption_gcm_err;
1051
1052         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
1053                 goto process_auth_encryption_gcm_err;
1054
1055         return 0;
1056
1057 process_auth_encryption_gcm_err:
1058         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
1059         return -EINVAL;
1060 }
1061
1062 /** Process AES-CCM encrypt algorithm */
1063 static int
1064 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1065                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1066                 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx)
1067 {
1068         int len = 0;
1069
1070         if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1071                 goto process_auth_encryption_ccm_err;
1072
1073         if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1074                 goto process_auth_encryption_ccm_err;
1075
1076         if (aadlen > 0)
1077                 /*
1078                  * For AES-CCM, the actual AAD is placed
1079                  * 18 bytes after the start of the AAD field,
1080                  * according to the API.
1081                  */
1082                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1083                         goto process_auth_encryption_ccm_err;
1084
1085         if (srclen > 0)
1086                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
1087                                 srclen, ctx))
1088                         goto process_auth_encryption_ccm_err;
1089
1090         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
1091                 goto process_auth_encryption_ccm_err;
1092
1093         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0)
1094                 goto process_auth_encryption_ccm_err;
1095
1096         return 0;
1097
1098 process_auth_encryption_ccm_err:
1099         OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed");
1100         return -EINVAL;
1101 }
1102
1103 /** Process AES-GCM decrypt algorithm */
1104 static int
1105 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
1106                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1107                 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx)
1108 {
1109         int len = 0, unused = 0;
1110         uint8_t empty[] = {};
1111
1112         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
1113                 goto process_auth_decryption_gcm_err;
1114
1115         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1116                 goto process_auth_decryption_gcm_err;
1117
1118         if (aadlen > 0)
1119                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
1120                         goto process_auth_decryption_gcm_err;
1121
1122         if (srclen > 0)
1123                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1124                                 srclen, ctx))
1125                         goto process_auth_decryption_gcm_err;
1126
1127         /* Workaround open ssl bug in version less then 1.0.1f */
1128         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
1129                 goto process_auth_decryption_gcm_err;
1130
1131         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
1132                 return -EFAULT;
1133
1134         return 0;
1135
1136 process_auth_decryption_gcm_err:
1137         OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed");
1138         return -EINVAL;
1139 }
1140
1141 /** Process AES-CCM decrypt algorithm */
1142 static int
1143 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset,
1144                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv,
1145                 uint8_t *dst, uint8_t *tag, uint8_t tag_len,
1146                 EVP_CIPHER_CTX *ctx)
1147 {
1148         int len = 0;
1149
1150         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0)
1151                 goto process_auth_decryption_ccm_err;
1152
1153         if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0)
1154                 goto process_auth_decryption_ccm_err;
1155
1156         if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0)
1157                 goto process_auth_decryption_ccm_err;
1158
1159         if (aadlen > 0)
1160                 /*
1161                  * For AES-CCM, the actual AAD is placed
1162                  * 18 bytes after the start of the AAD field,
1163                  * according to the API.
1164                  */
1165                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0)
1166                         goto process_auth_decryption_ccm_err;
1167
1168         if (srclen > 0)
1169                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
1170                                 srclen, ctx))
1171                         return -EFAULT;
1172
1173         return 0;
1174
1175 process_auth_decryption_ccm_err:
1176         OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed");
1177         return -EINVAL;
1178 }
1179
1180 /** Process standard openssl auth algorithms */
1181 static int
1182 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1183                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
1184                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
1185 {
1186         size_t dstlen;
1187         struct rte_mbuf *m;
1188         int l, n = srclen;
1189         uint8_t *src;
1190
1191         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1192                         m = m->next)
1193                 offset -= rte_pktmbuf_data_len(m);
1194
1195         if (m == 0)
1196                 goto process_auth_err;
1197
1198         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
1199                 goto process_auth_err;
1200
1201         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1202
1203         l = rte_pktmbuf_data_len(m) - offset;
1204         if (srclen <= l) {
1205                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
1206                         goto process_auth_err;
1207                 goto process_auth_final;
1208         }
1209
1210         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1211                 goto process_auth_err;
1212
1213         n -= l;
1214
1215         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1216                 src = rte_pktmbuf_mtod(m, uint8_t *);
1217                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1218                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
1219                         goto process_auth_err;
1220                 n -= l;
1221         }
1222
1223 process_auth_final:
1224         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
1225                 goto process_auth_err;
1226         return 0;
1227
1228 process_auth_err:
1229         OPENSSL_LOG_ERR("Process openssl auth failed");
1230         return -EINVAL;
1231 }
1232
1233 /** Process standard openssl auth algorithms with hmac */
1234 static int
1235 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
1236                 int srclen, HMAC_CTX *ctx)
1237 {
1238         unsigned int dstlen;
1239         struct rte_mbuf *m;
1240         int l, n = srclen;
1241         uint8_t *src;
1242
1243         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
1244                         m = m->next)
1245                 offset -= rte_pktmbuf_data_len(m);
1246
1247         if (m == 0)
1248                 goto process_auth_err;
1249
1250         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
1251
1252         l = rte_pktmbuf_data_len(m) - offset;
1253         if (srclen <= l) {
1254                 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1)
1255                         goto process_auth_err;
1256                 goto process_auth_final;
1257         }
1258
1259         if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1260                 goto process_auth_err;
1261
1262         n -= l;
1263
1264         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
1265                 src = rte_pktmbuf_mtod(m, uint8_t *);
1266                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
1267                 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1)
1268                         goto process_auth_err;
1269                 n -= l;
1270         }
1271
1272 process_auth_final:
1273         if (HMAC_Final(ctx, dst, &dstlen) != 1)
1274                 goto process_auth_err;
1275
1276         if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1))
1277                 goto process_auth_err;
1278
1279         return 0;
1280
1281 process_auth_err:
1282         OPENSSL_LOG_ERR("Process openssl auth failed");
1283         return -EINVAL;
1284 }
1285
1286 /*----------------------------------------------------------------------------*/
1287
1288 /** Process auth/cipher combined operation */
1289 static void
1290 process_openssl_combined_op
1291                 (struct rte_crypto_op *op, struct openssl_session *sess,
1292                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1293 {
1294         /* cipher */
1295         uint8_t *dst = NULL, *iv, *tag, *aad;
1296         int srclen, aadlen, status = -1;
1297         uint32_t offset;
1298         uint8_t taglen;
1299
1300         /*
1301          * Segmented destination buffer is not supported for
1302          * encryption/decryption
1303          */
1304         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1305                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1306                 return;
1307         }
1308
1309         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1310                         sess->iv.offset);
1311         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) {
1312                 srclen = 0;
1313                 offset = op->sym->auth.data.offset;
1314                 aadlen = op->sym->auth.data.length;
1315                 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1316                                 op->sym->auth.data.offset);
1317                 tag = op->sym->auth.digest.data;
1318                 if (tag == NULL)
1319                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1320                                 offset + aadlen);
1321         } else {
1322                 srclen = op->sym->aead.data.length;
1323                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1324                                 op->sym->aead.data.offset);
1325                 offset = op->sym->aead.data.offset;
1326                 aad = op->sym->aead.aad.data;
1327                 aadlen = sess->auth.aad_length;
1328                 tag = op->sym->aead.digest.data;
1329                 if (tag == NULL)
1330                         tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1331                                 offset + srclen);
1332         }
1333
1334         taglen = sess->auth.digest_length;
1335
1336         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1337                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1338                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1339                         status = process_openssl_auth_encryption_gcm(
1340                                         mbuf_src, offset, srclen,
1341                                         aad, aadlen, iv,
1342                                         dst, tag, sess->cipher.ctx);
1343                 else
1344                         status = process_openssl_auth_encryption_ccm(
1345                                         mbuf_src, offset, srclen,
1346                                         aad, aadlen, iv,
1347                                         dst, tag, taglen, sess->cipher.ctx);
1348
1349         } else {
1350                 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC ||
1351                                 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM)
1352                         status = process_openssl_auth_decryption_gcm(
1353                                         mbuf_src, offset, srclen,
1354                                         aad, aadlen, iv,
1355                                         dst, tag, sess->cipher.ctx);
1356                 else
1357                         status = process_openssl_auth_decryption_ccm(
1358                                         mbuf_src, offset, srclen,
1359                                         aad, aadlen, iv,
1360                                         dst, tag, taglen, sess->cipher.ctx);
1361         }
1362
1363         if (status != 0) {
1364                 if (status == (-EFAULT) &&
1365                                 sess->auth.operation ==
1366                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
1367                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1368                 else
1369                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1370         }
1371 }
1372
1373 /** Process cipher operation */
1374 static void
1375 process_openssl_cipher_op
1376                 (struct rte_crypto_op *op, struct openssl_session *sess,
1377                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1378 {
1379         uint8_t *dst, *iv;
1380         int srclen, status;
1381
1382         /*
1383          * Segmented destination buffer is not supported for
1384          * encryption/decryption
1385          */
1386         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
1387                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1388                 return;
1389         }
1390
1391         srclen = op->sym->cipher.data.length;
1392         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1393                         op->sym->cipher.data.offset);
1394
1395         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1396                         sess->iv.offset);
1397
1398         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
1399                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
1400                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1401                                         op->sym->cipher.data.offset, iv,
1402                                         srclen, sess->cipher.ctx);
1403                 else
1404                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1405                                         op->sym->cipher.data.offset, iv,
1406                                         srclen, sess->cipher.ctx);
1407         else
1408                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1409                                 op->sym->cipher.data.offset, iv,
1410                                 sess->cipher.key.data, srclen,
1411                                 sess->cipher.ctx);
1412
1413         if (status != 0)
1414                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1415 }
1416
1417 /** Process cipher operation */
1418 static void
1419 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1420                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1421                 struct rte_mbuf *mbuf_dst)
1422 {
1423         uint8_t *src, *dst, *iv;
1424         uint8_t block_size, last_block_len;
1425         int srclen, status = 0;
1426
1427         srclen = op->sym->cipher.data.length;
1428         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1429                         op->sym->cipher.data.offset);
1430         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1431                         op->sym->cipher.data.offset);
1432
1433         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1434                         sess->iv.offset);
1435
1436         block_size = DES_BLOCK_SIZE;
1437
1438         last_block_len = srclen % block_size;
1439         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1440                 /* Encrypt only with ECB mode XOR IV */
1441                 if (srclen < block_size) {
1442                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1443                                         iv, srclen,
1444                                         sess->cipher.bpi_ctx);
1445                 } else {
1446                         srclen -= last_block_len;
1447                         /* Encrypt with the block aligned stream with CBC mode */
1448                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1449                                         op->sym->cipher.data.offset, iv,
1450                                         srclen, sess->cipher.ctx);
1451                         if (last_block_len) {
1452                                 /* Point at last block */
1453                                 dst += srclen;
1454                                 /*
1455                                  * IV is the last encrypted block from
1456                                  * the previous operation
1457                                  */
1458                                 iv = dst - block_size;
1459                                 src += srclen;
1460                                 srclen = last_block_len;
1461                                 /* Encrypt the last frame with ECB mode */
1462                                 status |= process_openssl_cipher_bpi_encrypt(src,
1463                                                 dst, iv,
1464                                                 srclen, sess->cipher.bpi_ctx);
1465                         }
1466                 }
1467         } else {
1468                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1469                 if (srclen < block_size) {
1470                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1471                                         iv,
1472                                         srclen,
1473                                         sess->cipher.bpi_ctx);
1474                 } else {
1475                         if (last_block_len) {
1476                                 /* Point at last block */
1477                                 dst += srclen - last_block_len;
1478                                 src += srclen - last_block_len;
1479                                 /*
1480                                  * IV is the last full block
1481                                  */
1482                                 iv = src - block_size;
1483                                 /*
1484                                  * Decrypt the last frame with ECB mode
1485                                  * (encrypt, as it is the same operation)
1486                                  */
1487                                 status = process_openssl_cipher_bpi_encrypt(src,
1488                                                 dst, iv,
1489                                                 last_block_len, sess->cipher.bpi_ctx);
1490                                 /* Prepare parameters for CBC mode op */
1491                                 iv = rte_crypto_op_ctod_offset(op, uint8_t *,
1492                                                 sess->iv.offset);
1493                                 dst += last_block_len - srclen;
1494                                 srclen -= last_block_len;
1495                         }
1496
1497                         /* Decrypt with CBC mode */
1498                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1499                                         op->sym->cipher.data.offset, iv,
1500                                         srclen, sess->cipher.ctx);
1501                 }
1502         }
1503
1504         if (status != 0)
1505                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1506 }
1507
1508 /** Process auth operation */
1509 static void
1510 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1511                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1512                 struct rte_mbuf *mbuf_dst)
1513 {
1514         uint8_t *dst;
1515         int srclen, status;
1516
1517         srclen = op->sym->auth.data.length;
1518
1519         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1520                 dst = qp->temp_digest;
1521         else {
1522                 dst = op->sym->auth.digest.data;
1523                 if (dst == NULL)
1524                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1525                                         op->sym->auth.data.offset +
1526                                         op->sym->auth.data.length);
1527         }
1528
1529         switch (sess->auth.mode) {
1530         case OPENSSL_AUTH_AS_AUTH:
1531                 status = process_openssl_auth(mbuf_src, dst,
1532                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1533                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1534                 break;
1535         case OPENSSL_AUTH_AS_HMAC:
1536                 status = process_openssl_auth_hmac(mbuf_src, dst,
1537                                 op->sym->auth.data.offset, srclen,
1538                                 sess->auth.hmac.ctx);
1539                 break;
1540         default:
1541                 status = -1;
1542                 break;
1543         }
1544
1545         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1546                 if (memcmp(dst, op->sym->auth.digest.data,
1547                                 sess->auth.digest_length) != 0) {
1548                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1549                 }
1550         }
1551
1552         if (status != 0)
1553                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1554 }
1555
1556 /** Process crypto operation for mbuf */
1557 static int
1558 process_op(struct openssl_qp *qp, struct rte_crypto_op *op,
1559                 struct openssl_session *sess)
1560 {
1561         struct rte_mbuf *msrc, *mdst;
1562         int retval;
1563
1564         msrc = op->sym->m_src;
1565         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1566
1567         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1568
1569         switch (sess->chain_order) {
1570         case OPENSSL_CHAIN_ONLY_CIPHER:
1571                 process_openssl_cipher_op(op, sess, msrc, mdst);
1572                 break;
1573         case OPENSSL_CHAIN_ONLY_AUTH:
1574                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1575                 break;
1576         case OPENSSL_CHAIN_CIPHER_AUTH:
1577                 process_openssl_cipher_op(op, sess, msrc, mdst);
1578                 process_openssl_auth_op(qp, op, sess, mdst, mdst);
1579                 break;
1580         case OPENSSL_CHAIN_AUTH_CIPHER:
1581                 process_openssl_auth_op(qp, op, sess, msrc, mdst);
1582                 process_openssl_cipher_op(op, sess, msrc, mdst);
1583                 break;
1584         case OPENSSL_CHAIN_COMBINED:
1585                 process_openssl_combined_op(op, sess, msrc, mdst);
1586                 break;
1587         case OPENSSL_CHAIN_CIPHER_BPI:
1588                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1589                 break;
1590         default:
1591                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1592                 break;
1593         }
1594
1595         /* Free session if a session-less crypto op */
1596         if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) {
1597                 openssl_reset_session(sess);
1598                 memset(sess, 0, sizeof(struct openssl_session));
1599                 memset(op->sym->session, 0,
1600                                 rte_cryptodev_get_header_session_size());
1601                 rte_mempool_put(qp->sess_mp, sess);
1602                 rte_mempool_put(qp->sess_mp, op->sym->session);
1603                 op->sym->session = NULL;
1604         }
1605
1606         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1607                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1608
1609         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1610                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1611         else
1612                 retval = -1;
1613
1614         return retval;
1615 }
1616
1617 /*
1618  *------------------------------------------------------------------------------
1619  * PMD Framework
1620  *------------------------------------------------------------------------------
1621  */
1622
1623 /** Enqueue burst */
1624 static uint16_t
1625 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1626                 uint16_t nb_ops)
1627 {
1628         struct openssl_session *sess;
1629         struct openssl_qp *qp = queue_pair;
1630         int i, retval;
1631
1632         for (i = 0; i < nb_ops; i++) {
1633                 sess = get_session(qp, ops[i]);
1634                 if (unlikely(sess == NULL))
1635                         goto enqueue_err;
1636
1637                 retval = process_op(qp, ops[i], sess);
1638                 if (unlikely(retval < 0))
1639                         goto enqueue_err;
1640         }
1641
1642         qp->stats.enqueued_count += i;
1643         return i;
1644
1645 enqueue_err:
1646         qp->stats.enqueue_err_count++;
1647         return i;
1648 }
1649
1650 /** Dequeue burst */
1651 static uint16_t
1652 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1653                 uint16_t nb_ops)
1654 {
1655         struct openssl_qp *qp = queue_pair;
1656
1657         unsigned int nb_dequeued = 0;
1658
1659         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1660                         (void **)ops, nb_ops, NULL);
1661         qp->stats.dequeued_count += nb_dequeued;
1662
1663         return nb_dequeued;
1664 }
1665
1666 /** Create OPENSSL crypto device */
1667 static int
1668 cryptodev_openssl_create(const char *name,
1669                         struct rte_vdev_device *vdev,
1670                         struct rte_cryptodev_pmd_init_params *init_params)
1671 {
1672         struct rte_cryptodev *dev;
1673         struct openssl_private *internals;
1674
1675         dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params);
1676         if (dev == NULL) {
1677                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1678                 goto init_error;
1679         }
1680
1681         dev->driver_id = cryptodev_driver_id;
1682         dev->dev_ops = rte_openssl_pmd_ops;
1683
1684         /* register rx/tx burst functions for data path */
1685         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1686         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1687
1688         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1689                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1690                         RTE_CRYPTODEV_FF_CPU_AESNI |
1691                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1692
1693         /* Set vector instructions mode supported */
1694         internals = dev->data->dev_private;
1695
1696         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1697         internals->max_nb_sessions = init_params->max_nb_sessions;
1698
1699         return 0;
1700
1701 init_error:
1702         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1703                         init_params->name);
1704
1705         cryptodev_openssl_remove(vdev);
1706         return -EFAULT;
1707 }
1708
1709 /** Initialise OPENSSL crypto device */
1710 static int
1711 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1712 {
1713         struct rte_cryptodev_pmd_init_params init_params = {
1714                 "",
1715                 sizeof(struct openssl_private),
1716                 rte_socket_id(),
1717                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS,
1718                 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS
1719         };
1720         const char *name;
1721         const char *input_args;
1722
1723         name = rte_vdev_device_name(vdev);
1724         if (name == NULL)
1725                 return -EINVAL;
1726         input_args = rte_vdev_device_args(vdev);
1727
1728         rte_cryptodev_pmd_parse_input_args(&init_params, input_args);
1729
1730         return cryptodev_openssl_create(name, vdev, &init_params);
1731 }
1732
1733 /** Uninitialise OPENSSL crypto device */
1734 static int
1735 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1736 {
1737         struct rte_cryptodev *cryptodev;
1738         const char *name;
1739
1740         name = rte_vdev_device_name(vdev);
1741         if (name == NULL)
1742                 return -EINVAL;
1743
1744         cryptodev = rte_cryptodev_pmd_get_named_dev(name);
1745         if (cryptodev == NULL)
1746                 return -ENODEV;
1747
1748         return rte_cryptodev_pmd_destroy(cryptodev);
1749 }
1750
1751 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1752         .probe = cryptodev_openssl_probe,
1753         .remove = cryptodev_openssl_remove
1754 };
1755
1756 static struct cryptodev_driver openssl_crypto_drv;
1757
1758 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1759         cryptodev_openssl_pmd_drv);
1760 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1761         "max_nb_queue_pairs=<int> "
1762         "max_nb_sessions=<int> "
1763         "socket_id=<int>");
1764 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, cryptodev_openssl_pmd_drv,
1765                 cryptodev_driver_id);