drivers/crypto/openssl/rte_openssl_pmd.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2016 Intel Corporation. All rights reserved.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Intel Corporation nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <rte_common.h>
  34 #include <rte_hexdump.h>
  35 #include <rte_cryptodev.h>
  36 #include <rte_cryptodev_pmd.h>
  37 #include <rte_vdev.h>
  38 #include <rte_malloc.h>
  39 #include <rte_cpuflags.h>
  40
  41 #include <openssl/evp.h>
  42
  43 #include "rte_openssl_pmd_private.h"
  44
  45 #define DES_BLOCK_SIZE 8
  46
  47 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev);
  48
  49 /*----------------------------------------------------------------------------*/
  50
  51 /**
  52  * Increment counter by 1
  53  * Counter is 64 bit array, big-endian
  54  */
  55 static void
  56 ctr_inc(uint8_t *ctr)
  57 {
  58         uint64_t *ctr64 = (uint64_t *)ctr;
  59
  60         *ctr64 = __builtin_bswap64(*ctr64);
  61         (*ctr64)++;
  62         *ctr64 = __builtin_bswap64(*ctr64);
  63 }
  64
  65 /*
  66  *------------------------------------------------------------------------------
  67  * Session Prepare
  68  *------------------------------------------------------------------------------
  69  */
  70
  71 /** Get xform chain order */
  72 static enum openssl_chain_order
  73 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform)
  74 {
  75         enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED;
  76
  77         if (xform != NULL) {
  78                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
  79                         if (xform->next == NULL)
  80                                 res =  OPENSSL_CHAIN_ONLY_AUTH;
  81                         else if (xform->next->type ==
  82                                         RTE_CRYPTO_SYM_XFORM_CIPHER)
  83                                 res =  OPENSSL_CHAIN_AUTH_CIPHER;
  84                 }
  85                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
  86                         if (xform->next == NULL)
  87                                 res =  OPENSSL_CHAIN_ONLY_CIPHER;
  88                         else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
  89                                 res =  OPENSSL_CHAIN_CIPHER_AUTH;
  90                 }
  91         }
  92
  93         return res;
  94 }
  95
  96 /** Get session cipher key from input cipher key */
  97 static void
  98 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key)
  99 {
 100         memcpy(session_key, input_key, keylen);
 101 }
 102
 103 /** Get key ede 24 bytes standard from input key */
 104 static int
 105 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede)
 106 {
 107         int res = 0;
 108
 109         /* Initialize keys - 24 bytes: [key1-key2-key3] */
 110         switch (keylen) {
 111         case 24:
 112                 memcpy(key_ede, key, 24);
 113                 break;
 114         case 16:
 115                 /* K3 = K1 */
 116                 memcpy(key_ede, key, 16);
 117                 memcpy(key_ede + 16, key, 8);
 118                 break;
 119         case 8:
 120                 /* K1 = K2 = K3 (DES compatibility) */
 121                 memcpy(key_ede, key, 8);
 122                 memcpy(key_ede + 8, key, 8);
 123                 memcpy(key_ede + 16, key, 8);
 124                 break;
 125         default:
 126                 OPENSSL_LOG_ERR("Unsupported key size");
 127                 res = -EINVAL;
 128         }
 129
 130         return res;
 131 }
 132
 133 /** Get adequate openssl function for input cipher algorithm */
 134 static uint8_t
 135 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen,
 136                 const EVP_CIPHER **algo)
 137 {
 138         int res = 0;
 139
 140         if (algo != NULL) {
 141                 switch (sess_algo) {
 142                 case RTE_CRYPTO_CIPHER_3DES_CBC:
 143                         switch (keylen) {
 144                         case 16:
 145                                 *algo = EVP_des_ede_cbc();
 146                                 break;
 147                         case 24:
 148                                 *algo = EVP_des_ede3_cbc();
 149                                 break;
 150                         default:
 151                                 res = -EINVAL;
 152                         }
 153                         break;
 154                 case RTE_CRYPTO_CIPHER_3DES_CTR:
 155                         break;
 156                 case RTE_CRYPTO_CIPHER_AES_CBC:
 157                         switch (keylen) {
 158                         case 16:
 159                                 *algo = EVP_aes_128_cbc();
 160                                 break;
 161                         case 24:
 162                                 *algo = EVP_aes_192_cbc();
 163                                 break;
 164                         case 32:
 165                                 *algo = EVP_aes_256_cbc();
 166                                 break;
 167                         default:
 168                                 res = -EINVAL;
 169                         }
 170                         break;
 171                 case RTE_CRYPTO_CIPHER_AES_CTR:
 172                         switch (keylen) {
 173                         case 16:
 174                                 *algo = EVP_aes_128_ctr();
 175                                 break;
 176                         case 24:
 177                                 *algo = EVP_aes_192_ctr();
 178                                 break;
 179                         case 32:
 180                                 *algo = EVP_aes_256_ctr();
 181                                 break;
 182                         default:
 183                                 res = -EINVAL;
 184                         }
 185                         break;
 186                 case RTE_CRYPTO_CIPHER_AES_GCM:
 187                         switch (keylen) {
 188                         case 16:
 189                                 *algo = EVP_aes_128_gcm();
 190                                 break;
 191                         case 24:
 192                                 *algo = EVP_aes_192_gcm();
 193                                 break;
 194                         case 32:
 195                                 *algo = EVP_aes_256_gcm();
 196                                 break;
 197                         default:
 198                                 res = -EINVAL;
 199                         }
 200                         break;
 201                 default:
 202                         res = -EINVAL;
 203                         break;
 204                 }
 205         } else {
 206                 res = -EINVAL;
 207         }
 208
 209         return res;
 210 }
 211
 212 /** Get adequate openssl function for input auth algorithm */
 213 static uint8_t
 214 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo,
 215                 const EVP_MD **algo)
 216 {
 217         int res = 0;
 218
 219         if (algo != NULL) {
 220                 switch (sessalgo) {
 221                 case RTE_CRYPTO_AUTH_MD5:
 222                 case RTE_CRYPTO_AUTH_MD5_HMAC:
 223                         *algo = EVP_md5();
 224                         break;
 225                 case RTE_CRYPTO_AUTH_SHA1:
 226                 case RTE_CRYPTO_AUTH_SHA1_HMAC:
 227                         *algo = EVP_sha1();
 228                         break;
 229                 case RTE_CRYPTO_AUTH_SHA224:
 230                 case RTE_CRYPTO_AUTH_SHA224_HMAC:
 231                         *algo = EVP_sha224();
 232                         break;
 233                 case RTE_CRYPTO_AUTH_SHA256:
 234                 case RTE_CRYPTO_AUTH_SHA256_HMAC:
 235                         *algo = EVP_sha256();
 236                         break;
 237                 case RTE_CRYPTO_AUTH_SHA384:
 238                 case RTE_CRYPTO_AUTH_SHA384_HMAC:
 239                         *algo = EVP_sha384();
 240                         break;
 241                 case RTE_CRYPTO_AUTH_SHA512:
 242                 case RTE_CRYPTO_AUTH_SHA512_HMAC:
 243                         *algo = EVP_sha512();
 244                         break;
 245                 default:
 246                         res = -EINVAL;
 247                         break;
 248                 }
 249         } else {
 250                 res = -EINVAL;
 251         }
 252
 253         return res;
 254 }
 255
 256 /** Set session cipher parameters */
 257 static int
 258 openssl_set_session_cipher_parameters(struct openssl_session *sess,
 259                 const struct rte_crypto_sym_xform *xform)
 260 {
 261         /* Select cipher direction */
 262         sess->cipher.direction = xform->cipher.op;
 263         /* Select cipher key */
 264         sess->cipher.key.length = xform->cipher.key.length;
 265
 266         /* Select cipher algo */
 267         switch (xform->cipher.algo) {
 268         case RTE_CRYPTO_CIPHER_3DES_CBC:
 269         case RTE_CRYPTO_CIPHER_AES_CBC:
 270         case RTE_CRYPTO_CIPHER_AES_CTR:
 271         case RTE_CRYPTO_CIPHER_AES_GCM:
 272                 sess->cipher.mode = OPENSSL_CIPHER_LIB;
 273                 sess->cipher.algo = xform->cipher.algo;
 274                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 275
 276                 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length,
 277                                 &sess->cipher.evp_algo) != 0)
 278                         return -EINVAL;
 279
 280                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 281                         sess->cipher.key.data);
 282
 283                 break;
 284
 285         case RTE_CRYPTO_CIPHER_3DES_CTR:
 286                 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR;
 287                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 288
 289                 if (get_cipher_key_ede(xform->cipher.key.data,
 290                                 sess->cipher.key.length,
 291                                 sess->cipher.key.data) != 0)
 292                         return -EINVAL;
 293                 break;
 294         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 295                 sess->cipher.algo = xform->cipher.algo;
 296                 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI;
 297                 sess->cipher.ctx = EVP_CIPHER_CTX_new();
 298                 sess->cipher.evp_algo = EVP_des_cbc();
 299
 300                 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new();
 301                 /* IV will be ECB encrypted whether direction is encrypt or decrypt */
 302                 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(),
 303                                 NULL, xform->cipher.key.data, 0) != 1)
 304                         return -EINVAL;
 305
 306                 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length,
 307                         sess->cipher.key.data);
 308                 break;
 309         default:
 310                 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL;
 311                 return -EINVAL;
 312         }
 313
 314         return 0;
 315 }
 316
 317 /* Set session auth parameters */
 318 static int
 319 openssl_set_session_auth_parameters(struct openssl_session *sess,
 320                 const struct rte_crypto_sym_xform *xform)
 321 {
 322         /* Select auth generate/verify */
 323         sess->auth.operation = xform->auth.op;
 324         sess->auth.algo = xform->auth.algo;
 325
 326         /* Select auth algo */
 327         switch (xform->auth.algo) {
 328         case RTE_CRYPTO_AUTH_AES_GMAC:
 329         case RTE_CRYPTO_AUTH_AES_GCM:
 330                 /* Check additional condition for AES_GMAC/GCM */
 331                 if (sess->cipher.algo != RTE_CRYPTO_CIPHER_AES_GCM)
 332                         return -EINVAL;
 333                 sess->chain_order = OPENSSL_CHAIN_COMBINED;
 334                 break;
 335
 336         case RTE_CRYPTO_AUTH_MD5:
 337         case RTE_CRYPTO_AUTH_SHA1:
 338         case RTE_CRYPTO_AUTH_SHA224:
 339         case RTE_CRYPTO_AUTH_SHA256:
 340         case RTE_CRYPTO_AUTH_SHA384:
 341         case RTE_CRYPTO_AUTH_SHA512:
 342                 sess->auth.mode = OPENSSL_AUTH_AS_AUTH;
 343                 if (get_auth_algo(xform->auth.algo,
 344                                 &sess->auth.auth.evp_algo) != 0)
 345                         return -EINVAL;
 346                 sess->auth.auth.ctx = EVP_MD_CTX_create();
 347                 break;
 348
 349         case RTE_CRYPTO_AUTH_MD5_HMAC:
 350         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 351         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 352         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 353         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 354         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 355                 sess->auth.mode = OPENSSL_AUTH_AS_HMAC;
 356                 sess->auth.hmac.ctx = EVP_MD_CTX_create();
 357                 if (get_auth_algo(xform->auth.algo,
 358                                 &sess->auth.hmac.evp_algo) != 0)
 359                         return -EINVAL;
 360                 sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL,
 361                                 xform->auth.key.data, xform->auth.key.length);
 362                 break;
 363
 364         default:
 365                 return -EINVAL;
 366         }
 367
 368         return 0;
 369 }
 370
 371 /** Parse crypto xform chain and set private session parameters */
 372 int
 373 openssl_set_session_parameters(struct openssl_session *sess,
 374                 const struct rte_crypto_sym_xform *xform)
 375 {
 376         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 377         const struct rte_crypto_sym_xform *auth_xform = NULL;
 378
 379         sess->chain_order = openssl_get_chain_order(xform);
 380         switch (sess->chain_order) {
 381         case OPENSSL_CHAIN_ONLY_CIPHER:
 382                 cipher_xform = xform;
 383                 break;
 384         case OPENSSL_CHAIN_ONLY_AUTH:
 385                 auth_xform = xform;
 386                 break;
 387         case OPENSSL_CHAIN_CIPHER_AUTH:
 388                 cipher_xform = xform;
 389                 auth_xform = xform->next;
 390                 break;
 391         case OPENSSL_CHAIN_AUTH_CIPHER:
 392                 auth_xform = xform;
 393                 cipher_xform = xform->next;
 394                 break;
 395         default:
 396                 return -EINVAL;
 397         }
 398
 399         /* cipher_xform must be check before auth_xform */
 400         if (cipher_xform) {
 401                 if (openssl_set_session_cipher_parameters(
 402                                 sess, cipher_xform)) {
 403                         OPENSSL_LOG_ERR(
 404                                 "Invalid/unsupported cipher parameters");
 405                         return -EINVAL;
 406                 }
 407         }
 408
 409         if (auth_xform) {
 410                 if (openssl_set_session_auth_parameters(sess, auth_xform)) {
 411                         OPENSSL_LOG_ERR(
 412                                 "Invalid/unsupported auth parameters");
 413                         return -EINVAL;
 414                 }
 415         }
 416
 417         return 0;
 418 }
 419
 420 /** Reset private session parameters */
 421 void
 422 openssl_reset_session(struct openssl_session *sess)
 423 {
 424         EVP_CIPHER_CTX_free(sess->cipher.ctx);
 425
 426         if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI)
 427                 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx);
 428
 429         switch (sess->auth.mode) {
 430         case OPENSSL_AUTH_AS_AUTH:
 431                 EVP_MD_CTX_destroy(sess->auth.auth.ctx);
 432                 break;
 433         case OPENSSL_AUTH_AS_HMAC:
 434                 EVP_PKEY_free(sess->auth.hmac.pkey);
 435                 EVP_MD_CTX_destroy(sess->auth.hmac.ctx);
 436                 break;
 437         default:
 438                 break;
 439         }
 440 }
 441
 442 /** Provide session for operation */
 443 static struct openssl_session *
 444 get_session(struct openssl_qp *qp, struct rte_crypto_op *op)
 445 {
 446         struct openssl_session *sess = NULL;
 447
 448         if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) {
 449                 /* get existing session */
 450                 if (likely(op->sym->session != NULL &&
 451                                 op->sym->session->dev_type ==
 452                                 RTE_CRYPTODEV_OPENSSL_PMD))
 453                         sess = (struct openssl_session *)
 454                                 op->sym->session->_private;
 455         } else  {
 456                 /* provide internal session */
 457                 void *_sess = NULL;
 458
 459                 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) {
 460                         sess = (struct openssl_session *)
 461                                 ((struct rte_cryptodev_sym_session *)_sess)
 462                                 ->_private;
 463
 464                         if (unlikely(openssl_set_session_parameters(
 465                                         sess, op->sym->xform) != 0)) {
 466                                 rte_mempool_put(qp->sess_mp, _sess);
 467                                 sess = NULL;
 468                         } else
 469                                 op->sym->session = _sess;
 470                 }
 471         }
 472
 473         if (sess == NULL)
 474                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 475
 476         return sess;
 477 }
 478
 479 /*
 480  *------------------------------------------------------------------------------
 481  * Process Operations
 482  *------------------------------------------------------------------------------
 483  */
 484 static inline int
 485 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset,
 486                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 487 {
 488         struct rte_mbuf *m;
 489         int dstlen;
 490         int l, n = srclen;
 491         uint8_t *src;
 492
 493         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 494                         m = m->next)
 495                 offset -= rte_pktmbuf_data_len(m);
 496
 497         if (m == 0)
 498                 return -1;
 499
 500         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 501
 502         l = rte_pktmbuf_data_len(m) - offset;
 503         if (srclen <= l) {
 504                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 505                         return -1;
 506                 *dst += l;
 507                 return 0;
 508         }
 509
 510         if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 511                 return -1;
 512
 513         *dst += dstlen;
 514         n -= l;
 515
 516         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 517                 src = rte_pktmbuf_mtod(m, uint8_t *);
 518                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 519                 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 520                         return -1;
 521                 *dst += dstlen;
 522                 n -= l;
 523         }
 524
 525         return 0;
 526 }
 527
 528 static inline int
 529 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset,
 530                 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx)
 531 {
 532         struct rte_mbuf *m;
 533         int dstlen;
 534         int l, n = srclen;
 535         uint8_t *src;
 536
 537         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 538                         m = m->next)
 539                 offset -= rte_pktmbuf_data_len(m);
 540
 541         if (m == 0)
 542                 return -1;
 543
 544         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 545
 546         l = rte_pktmbuf_data_len(m) - offset;
 547         if (srclen <= l) {
 548                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0)
 549                         return -1;
 550                 *dst += l;
 551                 return 0;
 552         }
 553
 554         if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 555                 return -1;
 556
 557         *dst += dstlen;
 558         n -= l;
 559
 560         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 561                 src = rte_pktmbuf_mtod(m, uint8_t *);
 562                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 563                 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0)
 564                         return -1;
 565                 *dst += dstlen;
 566                 n -= l;
 567         }
 568
 569         return 0;
 570 }
 571
 572 /** Process standard openssl cipher encryption */
 573 static int
 574 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 575                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 576                 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
 577 {
 578         int totlen;
 579
 580         if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0)
 581                 goto process_cipher_encrypt_err;
 582
 583         EVP_CIPHER_CTX_set_padding(ctx, 0);
 584
 585         if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 586                         srclen, ctx))
 587                 goto process_cipher_encrypt_err;
 588
 589         if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0)
 590                 goto process_cipher_encrypt_err;
 591
 592         return 0;
 593
 594 process_cipher_encrypt_err:
 595         OPENSSL_LOG_ERR("Process openssl cipher encrypt failed");
 596         return -EINVAL;
 597 }
 598
 599 /** Process standard openssl cipher encryption */
 600 static int
 601 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst,
 602                 uint8_t *iv, int srclen,
 603                 EVP_CIPHER_CTX *ctx)
 604 {
 605         uint8_t i;
 606         uint8_t encrypted_iv[DES_BLOCK_SIZE];
 607         int encrypted_ivlen;
 608
 609         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen,
 610                         iv, DES_BLOCK_SIZE) <= 0)
 611                 goto process_cipher_encrypt_err;
 612
 613         for (i = 0; i < srclen; i++)
 614                 *(dst + i) = *(src + i) ^ (encrypted_iv[i]);
 615
 616         return 0;
 617
 618 process_cipher_encrypt_err:
 619         OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed");
 620         return -EINVAL;
 621 }
 622 /** Process standard openssl cipher decryption */
 623 static int
 624 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst,
 625                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 626                 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
 627 {
 628         int totlen;
 629
 630         if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0)
 631                 goto process_cipher_decrypt_err;
 632
 633         EVP_CIPHER_CTX_set_padding(ctx, 0);
 634
 635         if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 636                         srclen, ctx))
 637                 goto process_cipher_decrypt_err;
 638
 639         if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0)
 640                 goto process_cipher_decrypt_err;
 641         return 0;
 642
 643 process_cipher_decrypt_err:
 644         OPENSSL_LOG_ERR("Process openssl cipher decrypt failed");
 645         return -EINVAL;
 646 }
 647
 648 /** Process cipher des 3 ctr encryption, decryption algorithm */
 649 static int
 650 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst,
 651                 int offset, uint8_t *iv, uint8_t *key, int srclen,
 652                 EVP_CIPHER_CTX *ctx)
 653 {
 654         uint8_t ebuf[8], ctr[8];
 655         int unused, n;
 656         struct rte_mbuf *m;
 657         uint8_t *src;
 658         int l;
 659
 660         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 661                         m = m->next)
 662                 offset -= rte_pktmbuf_data_len(m);
 663
 664         if (m == 0)
 665                 goto process_cipher_des3ctr_err;
 666
 667         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 668         l = rte_pktmbuf_data_len(m) - offset;
 669
 670         /* We use 3DES encryption also for decryption.
 671          * IV is not important for 3DES ecb
 672          */
 673         if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0)
 674                 goto process_cipher_des3ctr_err;
 675
 676         memcpy(ctr, iv, 8);
 677
 678         for (n = 0; n < srclen; n++) {
 679                 if (n % 8 == 0) {
 680                         if (EVP_EncryptUpdate(ctx,
 681                                         (unsigned char *)&ebuf, &unused,
 682                                         (const unsigned char *)&ctr, 8) <= 0)
 683                                 goto process_cipher_des3ctr_err;
 684                         ctr_inc(ctr);
 685                 }
 686                 dst[n] = *(src++) ^ ebuf[n % 8];
 687
 688                 l--;
 689                 if (!l) {
 690                         m = m->next;
 691                         if (m) {
 692                                 src = rte_pktmbuf_mtod(m, uint8_t *);
 693                                 l = rte_pktmbuf_data_len(m);
 694                         }
 695                 }
 696         }
 697
 698         return 0;
 699
 700 process_cipher_des3ctr_err:
 701         OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed");
 702         return -EINVAL;
 703 }
 704
 705 /** Process auth/encription aes-gcm algorithm */
 706 static int
 707 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 708                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen,
 709                 uint8_t *key, uint8_t *dst, uint8_t *tag,
 710                 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo)
 711 {
 712         int len = 0, unused = 0;
 713         uint8_t empty[] = {};
 714
 715         if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0)
 716                 goto process_auth_encryption_gcm_err;
 717
 718         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0)
 719                 goto process_auth_encryption_gcm_err;
 720
 721         if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0)
 722                 goto process_auth_encryption_gcm_err;
 723
 724         if (aadlen > 0)
 725                 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
 726                         goto process_auth_encryption_gcm_err;
 727
 728         if (srclen > 0)
 729                 if (process_openssl_encryption_update(mbuf_src, offset, &dst,
 730                                 srclen, ctx))
 731                         goto process_auth_encryption_gcm_err;
 732
 733         /* Workaround open ssl bug in version less then 1.0.1f */
 734         if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
 735                 goto process_auth_encryption_gcm_err;
 736
 737         if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0)
 738                 goto process_auth_encryption_gcm_err;
 739
 740         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0)
 741                 goto process_auth_encryption_gcm_err;
 742
 743         return 0;
 744
 745 process_auth_encryption_gcm_err:
 746         OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed");
 747         return -EINVAL;
 748 }
 749
 750 static int
 751 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset,
 752                 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen,
 753                 uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx,
 754                 const EVP_CIPHER *algo)
 755 {
 756         int len = 0, unused = 0;
 757         uint8_t empty[] = {};
 758
 759         if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0)
 760                 goto process_auth_decryption_gcm_err;
 761
 762         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0)
 763                 goto process_auth_decryption_gcm_err;
 764
 765         if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0)
 766                 goto process_auth_decryption_gcm_err;
 767
 768         if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0)
 769                 goto process_auth_decryption_gcm_err;
 770
 771         if (aadlen > 0)
 772                 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0)
 773                         goto process_auth_decryption_gcm_err;
 774
 775         if (srclen > 0)
 776                 if (process_openssl_decryption_update(mbuf_src, offset, &dst,
 777                                 srclen, ctx))
 778                         goto process_auth_decryption_gcm_err;
 779
 780         /* Workaround open ssl bug in version less then 1.0.1f */
 781         if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0)
 782                 goto process_auth_decryption_gcm_err;
 783
 784         if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0)
 785                 goto process_auth_decryption_gcm_final_err;
 786
 787         return 0;
 788
 789 process_auth_decryption_gcm_err:
 790         OPENSSL_LOG_ERR("Process openssl auth description gcm failed");
 791         return -EINVAL;
 792
 793 process_auth_decryption_gcm_final_err:
 794         return -EFAULT;
 795 }
 796
 797 /** Process standard openssl auth algorithms */
 798 static int
 799 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
 800                 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey,
 801                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
 802 {
 803         size_t dstlen;
 804         struct rte_mbuf *m;
 805         int l, n = srclen;
 806         uint8_t *src;
 807
 808         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 809                         m = m->next)
 810                 offset -= rte_pktmbuf_data_len(m);
 811
 812         if (m == 0)
 813                 goto process_auth_err;
 814
 815         if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0)
 816                 goto process_auth_err;
 817
 818         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 819
 820         l = rte_pktmbuf_data_len(m) - offset;
 821         if (srclen <= l) {
 822                 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0)
 823                         goto process_auth_err;
 824                 goto process_auth_final;
 825         }
 826
 827         if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
 828                 goto process_auth_err;
 829
 830         n -= l;
 831
 832         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 833                 src = rte_pktmbuf_mtod(m, uint8_t *);
 834                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 835                 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0)
 836                         goto process_auth_err;
 837                 n -= l;
 838         }
 839
 840 process_auth_final:
 841         if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0)
 842                 goto process_auth_err;
 843         return 0;
 844
 845 process_auth_err:
 846         OPENSSL_LOG_ERR("Process openssl auth failed");
 847         return -EINVAL;
 848 }
 849
 850 /** Process standard openssl auth algorithms with hmac */
 851 static int
 852 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset,
 853                 __rte_unused uint8_t *iv, EVP_PKEY *pkey,
 854                 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo)
 855 {
 856         size_t dstlen;
 857         struct rte_mbuf *m;
 858         int l, n = srclen;
 859         uint8_t *src;
 860
 861         for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m);
 862                         m = m->next)
 863                 offset -= rte_pktmbuf_data_len(m);
 864
 865         if (m == 0)
 866                 goto process_auth_err;
 867
 868         if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0)
 869                 goto process_auth_err;
 870
 871         src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset);
 872
 873         l = rte_pktmbuf_data_len(m) - offset;
 874         if (srclen <= l) {
 875                 if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0)
 876                         goto process_auth_err;
 877                 goto process_auth_final;
 878         }
 879
 880         if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0)
 881                 goto process_auth_err;
 882
 883         n -= l;
 884
 885         for (m = m->next; (m != NULL) && (n > 0); m = m->next) {
 886                 src = rte_pktmbuf_mtod(m, uint8_t *);
 887                 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n;
 888                 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0)
 889                         goto process_auth_err;
 890                 n -= l;
 891         }
 892
 893 process_auth_final:
 894         if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0)
 895                 goto process_auth_err;
 896
 897         return 0;
 898
 899 process_auth_err:
 900         OPENSSL_LOG_ERR("Process openssl auth failed");
 901         return -EINVAL;
 902 }
 903
 904 /*----------------------------------------------------------------------------*/
 905
 906 /** Process auth/cipher combined operation */
 907 static void
 908 process_openssl_combined_op
 909                 (struct rte_crypto_op *op, struct openssl_session *sess,
 910                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
 911 {
 912         /* cipher */
 913         uint8_t *dst = NULL, *iv, *tag, *aad;
 914         int srclen, ivlen, aadlen, status = -1;
 915
 916         /*
 917          * Segmented destination buffer is not supported for
 918          * encryption/decryption
 919          */
 920         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
 921                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 922                 return;
 923         }
 924
 925         iv = op->sym->cipher.iv.data;
 926         ivlen = op->sym->cipher.iv.length;
 927         aad = op->sym->auth.aad.data;
 928         aadlen = op->sym->auth.aad.length;
 929
 930         tag = op->sym->auth.digest.data;
 931         if (tag == NULL)
 932                 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
 933                                 op->sym->cipher.data.offset +
 934                                 op->sym->cipher.data.length);
 935
 936         if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC)
 937                 srclen = 0;
 938         else {
 939                 srclen = op->sym->cipher.data.length;
 940                 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
 941                                 op->sym->cipher.data.offset);
 942         }
 943
 944         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 945                 status = process_openssl_auth_encryption_gcm(
 946                                 mbuf_src, op->sym->cipher.data.offset, srclen,
 947                                 aad, aadlen, iv, ivlen, sess->cipher.key.data,
 948                                 dst, tag, sess->cipher.ctx,
 949                                 sess->cipher.evp_algo);
 950         else
 951                 status = process_openssl_auth_decryption_gcm(
 952                                 mbuf_src, op->sym->cipher.data.offset, srclen,
 953                                 aad, aadlen, iv, ivlen, sess->cipher.key.data,
 954                                 dst, tag, sess->cipher.ctx,
 955                                 sess->cipher.evp_algo);
 956
 957         if (status != 0) {
 958                 if (status == (-EFAULT) &&
 959                                 sess->auth.operation ==
 960                                                 RTE_CRYPTO_AUTH_OP_VERIFY)
 961                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 962                 else
 963                         op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 964         }
 965 }
 966
 967 /** Process cipher operation */
 968 static void
 969 process_openssl_cipher_op
 970                 (struct rte_crypto_op *op, struct openssl_session *sess,
 971                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
 972 {
 973         uint8_t *dst, *iv;
 974         int srclen, status;
 975
 976         /*
 977          * Segmented destination buffer is not supported for
 978          * encryption/decryption
 979          */
 980         if (!rte_pktmbuf_is_contiguous(mbuf_dst)) {
 981                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 982                 return;
 983         }
 984
 985         srclen = op->sym->cipher.data.length;
 986         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
 987                         op->sym->cipher.data.offset);
 988
 989         iv = op->sym->cipher.iv.data;
 990
 991         if (sess->cipher.mode == OPENSSL_CIPHER_LIB)
 992                 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 993                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
 994                                         op->sym->cipher.data.offset, iv,
 995                                         sess->cipher.key.data, srclen,
 996                                         sess->cipher.ctx,
 997                                         sess->cipher.evp_algo);
 998                 else
 999                         status = process_openssl_cipher_decrypt(mbuf_src, dst,
1000                                         op->sym->cipher.data.offset, iv,
1001                                         sess->cipher.key.data, srclen,
1002                                         sess->cipher.ctx,
1003                                         sess->cipher.evp_algo);
1004         else
1005                 status = process_openssl_cipher_des3ctr(mbuf_src, dst,
1006                                 op->sym->cipher.data.offset, iv,
1007                                 sess->cipher.key.data, srclen,
1008                                 sess->cipher.ctx);
1009
1010         if (status != 0)
1011                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1012 }
1013
1014 /** Process cipher operation */
1015 static void
1016 process_openssl_docsis_bpi_op(struct rte_crypto_op *op,
1017                 struct openssl_session *sess, struct rte_mbuf *mbuf_src,
1018                 struct rte_mbuf *mbuf_dst)
1019 {
1020         uint8_t *src, *dst, *iv;
1021         uint8_t block_size, last_block_len;
1022         int srclen, status = 0;
1023
1024         srclen = op->sym->cipher.data.length;
1025         src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
1026                         op->sym->cipher.data.offset);
1027         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1028                         op->sym->cipher.data.offset);
1029
1030         iv = op->sym->cipher.iv.data;
1031
1032         block_size = DES_BLOCK_SIZE;
1033
1034         last_block_len = srclen % block_size;
1035         if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) {
1036                 /* Encrypt only with ECB mode XOR IV */
1037                 if (srclen < block_size) {
1038                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1039                                         iv, srclen,
1040                                         sess->cipher.bpi_ctx);
1041                 } else {
1042                         srclen -= last_block_len;
1043                         /* Encrypt with the block aligned stream with CBC mode */
1044                         status = process_openssl_cipher_encrypt(mbuf_src, dst,
1045                                         op->sym->cipher.data.offset, iv,
1046                                         sess->cipher.key.data, srclen,
1047                                         sess->cipher.ctx, sess->cipher.evp_algo);
1048                         if (last_block_len) {
1049                                 /* Point at last block */
1050                                 dst += srclen;
1051                                 /*
1052                                  * IV is the last encrypted block from
1053                                  * the previous operation
1054                                  */
1055                                 iv = dst - block_size;
1056                                 src += srclen;
1057                                 srclen = last_block_len;
1058                                 /* Encrypt the last frame with ECB mode */
1059                                 status |= process_openssl_cipher_bpi_encrypt(src,
1060                                                 dst, iv,
1061                                                 srclen, sess->cipher.bpi_ctx);
1062                         }
1063                 }
1064         } else {
1065                 /* Decrypt only with ECB mode (encrypt, as it is same operation) */
1066                 if (srclen < block_size) {
1067                         status = process_openssl_cipher_bpi_encrypt(src, dst,
1068                                         iv,
1069                                         srclen,
1070                                         sess->cipher.bpi_ctx);
1071                 } else {
1072                         if (last_block_len) {
1073                                 /* Point at last block */
1074                                 dst += srclen - last_block_len;
1075                                 src += srclen - last_block_len;
1076                                 /*
1077                                  * IV is the last full block
1078                                  */
1079                                 iv = src - block_size;
1080                                 /*
1081                                  * Decrypt the last frame with ECB mode
1082                                  * (encrypt, as it is the same operation)
1083                                  */
1084                                 status = process_openssl_cipher_bpi_encrypt(src,
1085                                                 dst, iv,
1086                                                 last_block_len, sess->cipher.bpi_ctx);
1087                                 /* Prepare parameters for CBC mode op */
1088                                 iv = op->sym->cipher.iv.data;
1089                                 dst += last_block_len - srclen;
1090                                 srclen -= last_block_len;
1091                         }
1092
1093                         /* Decrypt with CBC mode */
1094                         status |= process_openssl_cipher_decrypt(mbuf_src, dst,
1095                                         op->sym->cipher.data.offset, iv,
1096                                         sess->cipher.key.data, srclen,
1097                                         sess->cipher.ctx,
1098                                         sess->cipher.evp_algo);
1099                 }
1100         }
1101
1102         if (status != 0)
1103                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1104 }
1105
1106 /** Process auth operation */
1107 static void
1108 process_openssl_auth_op
1109                 (struct rte_crypto_op *op, struct openssl_session *sess,
1110                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
1111 {
1112         uint8_t *dst;
1113         int srclen, status;
1114
1115         srclen = op->sym->auth.data.length;
1116
1117         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY)
1118                 dst = (uint8_t *)rte_pktmbuf_append(mbuf_src,
1119                                 op->sym->auth.digest.length);
1120         else {
1121                 dst = op->sym->auth.digest.data;
1122                 if (dst == NULL)
1123                         dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
1124                                         op->sym->auth.data.offset +
1125                                         op->sym->auth.data.length);
1126         }
1127
1128         switch (sess->auth.mode) {
1129         case OPENSSL_AUTH_AS_AUTH:
1130                 status = process_openssl_auth(mbuf_src, dst,
1131                                 op->sym->auth.data.offset, NULL, NULL, srclen,
1132                                 sess->auth.auth.ctx, sess->auth.auth.evp_algo);
1133                 break;
1134         case OPENSSL_AUTH_AS_HMAC:
1135                 status = process_openssl_auth_hmac(mbuf_src, dst,
1136                                 op->sym->auth.data.offset, NULL,
1137                                 sess->auth.hmac.pkey, srclen,
1138                                 sess->auth.hmac.ctx, sess->auth.hmac.evp_algo);
1139                 break;
1140         default:
1141                 status = -1;
1142                 break;
1143         }
1144
1145         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
1146                 if (memcmp(dst, op->sym->auth.digest.data,
1147                                 op->sym->auth.digest.length) != 0) {
1148                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1149                 }
1150                 /* Trim area used for digest from mbuf. */
1151                 rte_pktmbuf_trim(mbuf_src, op->sym->auth.digest.length);
1152         }
1153
1154         if (status != 0)
1155                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1156 }
1157
1158 /** Process crypto operation for mbuf */
1159 static int
1160 process_op(const struct openssl_qp *qp, struct rte_crypto_op *op,
1161                 struct openssl_session *sess)
1162 {
1163         struct rte_mbuf *msrc, *mdst;
1164         int retval;
1165
1166         msrc = op->sym->m_src;
1167         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
1168
1169         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
1170
1171         switch (sess->chain_order) {
1172         case OPENSSL_CHAIN_ONLY_CIPHER:
1173                 process_openssl_cipher_op(op, sess, msrc, mdst);
1174                 break;
1175         case OPENSSL_CHAIN_ONLY_AUTH:
1176                 process_openssl_auth_op(op, sess, msrc, mdst);
1177                 break;
1178         case OPENSSL_CHAIN_CIPHER_AUTH:
1179                 process_openssl_cipher_op(op, sess, msrc, mdst);
1180                 process_openssl_auth_op(op, sess, mdst, mdst);
1181                 break;
1182         case OPENSSL_CHAIN_AUTH_CIPHER:
1183                 process_openssl_auth_op(op, sess, msrc, mdst);
1184                 process_openssl_cipher_op(op, sess, msrc, mdst);
1185                 break;
1186         case OPENSSL_CHAIN_COMBINED:
1187                 process_openssl_combined_op(op, sess, msrc, mdst);
1188                 break;
1189         case OPENSSL_CHAIN_CIPHER_BPI:
1190                 process_openssl_docsis_bpi_op(op, sess, msrc, mdst);
1191                 break;
1192         default:
1193                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
1194                 break;
1195         }
1196
1197         /* Free session if a session-less crypto op */
1198         if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) {
1199                 openssl_reset_session(sess);
1200                 memset(sess, 0, sizeof(struct openssl_session));
1201                 rte_mempool_put(qp->sess_mp, op->sym->session);
1202                 op->sym->session = NULL;
1203         }
1204
1205         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
1206                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1207
1208         if (op->status != RTE_CRYPTO_OP_STATUS_ERROR)
1209                 retval = rte_ring_enqueue(qp->processed_ops, (void *)op);
1210         else
1211                 retval = -1;
1212
1213         return retval;
1214 }
1215
1216 /*
1217  *------------------------------------------------------------------------------
1218  * PMD Framework
1219  *------------------------------------------------------------------------------
1220  */
1221
1222 /** Enqueue burst */
1223 static uint16_t
1224 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
1225                 uint16_t nb_ops)
1226 {
1227         struct openssl_session *sess;
1228         struct openssl_qp *qp = queue_pair;
1229         int i, retval;
1230
1231         for (i = 0; i < nb_ops; i++) {
1232                 sess = get_session(qp, ops[i]);
1233                 if (unlikely(sess == NULL))
1234                         goto enqueue_err;
1235
1236                 retval = process_op(qp, ops[i], sess);
1237                 if (unlikely(retval < 0))
1238                         goto enqueue_err;
1239         }
1240
1241         qp->stats.enqueued_count += i;
1242         return i;
1243
1244 enqueue_err:
1245         qp->stats.enqueue_err_count++;
1246         return i;
1247 }
1248
1249 /** Dequeue burst */
1250 static uint16_t
1251 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
1252                 uint16_t nb_ops)
1253 {
1254         struct openssl_qp *qp = queue_pair;
1255
1256         unsigned int nb_dequeued = 0;
1257
1258         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
1259                         (void **)ops, nb_ops, NULL);
1260         qp->stats.dequeued_count += nb_dequeued;
1261
1262         return nb_dequeued;
1263 }
1264
1265 /** Create OPENSSL crypto device */
1266 static int
1267 cryptodev_openssl_create(const char *name,
1268                         struct rte_vdev_device *vdev,
1269                         struct rte_crypto_vdev_init_params *init_params)
1270 {
1271         struct rte_cryptodev *dev;
1272         struct openssl_private *internals;
1273
1274         if (init_params->name[0] == '\0')
1275                 snprintf(init_params->name, sizeof(init_params->name),
1276                                 "%s", name);
1277
1278         dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name,
1279                         sizeof(struct openssl_private),
1280                         init_params->socket_id);
1281         if (dev == NULL) {
1282                 OPENSSL_LOG_ERR("failed to create cryptodev vdev");
1283                 goto init_error;
1284         }
1285
1286         dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD;
1287         dev->dev_ops = rte_openssl_pmd_ops;
1288
1289         /* register rx/tx burst functions for data path */
1290         dev->dequeue_burst = openssl_pmd_dequeue_burst;
1291         dev->enqueue_burst = openssl_pmd_enqueue_burst;
1292
1293         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
1294                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
1295                         RTE_CRYPTODEV_FF_CPU_AESNI |
1296                         RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER;
1297
1298         /* Set vector instructions mode supported */
1299         internals = dev->data->dev_private;
1300
1301         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
1302         internals->max_nb_sessions = init_params->max_nb_sessions;
1303
1304         return 0;
1305
1306 init_error:
1307         OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed",
1308                         init_params->name);
1309
1310         cryptodev_openssl_remove(vdev);
1311         return -EFAULT;
1312 }
1313
1314 /** Initialise OPENSSL crypto device */
1315 static int
1316 cryptodev_openssl_probe(struct rte_vdev_device *vdev)
1317 {
1318         struct rte_crypto_vdev_init_params init_params = {
1319                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
1320                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
1321                 rte_socket_id(),
1322                 {0}
1323         };
1324         const char *name;
1325         const char *input_args;
1326
1327         name = rte_vdev_device_name(vdev);
1328         if (name == NULL)
1329                 return -EINVAL;
1330         input_args = rte_vdev_device_args(vdev);
1331
1332         rte_cryptodev_parse_vdev_init_params(&init_params, input_args);
1333
1334         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
1335                         init_params.socket_id);
1336         if (init_params.name[0] != '\0')
1337                 RTE_LOG(INFO, PMD, "  User defined name = %s\n",
1338                         init_params.name);
1339         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
1340                         init_params.max_nb_queue_pairs);
1341         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
1342                         init_params.max_nb_sessions);
1343
1344         return cryptodev_openssl_create(name, vdev, &init_params);
1345 }
1346
1347 /** Uninitialise OPENSSL crypto device */
1348 static int
1349 cryptodev_openssl_remove(struct rte_vdev_device *vdev)
1350 {
1351         const char *name;
1352
1353         name = rte_vdev_device_name(vdev);
1354         if (name == NULL)
1355                 return -EINVAL;
1356
1357         RTE_LOG(INFO, PMD,
1358                 "Closing OPENSSL crypto device %s on numa socket %u\n",
1359                 name, rte_socket_id());
1360
1361         return 0;
1362 }
1363
1364 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = {
1365         .probe = cryptodev_openssl_probe,
1366         .remove = cryptodev_openssl_remove
1367 };
1368
1369 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD,
1370         cryptodev_openssl_pmd_drv);
1371 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD,
1372         "max_nb_queue_pairs=<int> "
1373         "max_nb_sessions=<int> "
1374         "socket_id=<int>");