drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2017 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_tailq.h>
  48 #include <rte_malloc.h>
  49 #include <rte_launch.h>
  50 #include <rte_eal.h>
  51 #include <rte_per_lcore.h>
  52 #include <rte_lcore.h>
  53 #include <rte_branch_prediction.h>
  54 #include <rte_mempool.h>
  55 #include <rte_mbuf.h>
  56 #include <rte_string_fns.h>
  57 #include <rte_spinlock.h>
  58 #include <rte_hexdump.h>
  59 #include <rte_crypto_sym.h>
  60 #include <rte_byteorder.h>
  61 #include <rte_pci.h>
  62 #include <rte_bus_pci.h>
  63
  64 #include <openssl/evp.h>
  65
  66 #include "qat_logs.h"
  67 #include "qat_algs.h"
  68 #include "qat_crypto.h"
  69 #include "adf_transport_access_macros.h"
  70
  71 #define BYTE_LENGTH    8
  72 /* bpi is only used for partial blocks of DES and AES
  73  * so AES block len can be assumed as max len for iv, src and dst
  74  */
  75 #define BPI_MAX_ENCR_IV_LEN ICP_QAT_HW_AES_BLK_SZ
  76
  77 static int
  78 qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
  79                 struct qat_pmd_private *internals) {
  80         int i = 0;
  81         const struct rte_cryptodev_capabilities *capability;
  82
  83         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  84                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
  85                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
  86                         continue;
  87
  88                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
  89                         continue;
  90
  91                 if (capability->sym.cipher.algo == algo)
  92                         return 1;
  93         }
  94         return 0;
  95 }
  96
  97 static int
  98 qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
  99                 struct qat_pmd_private *internals) {
 100         int i = 0;
 101         const struct rte_cryptodev_capabilities *capability;
 102
 103         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
 104                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 105                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 106                         continue;
 107
 108                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
 109                         continue;
 110
 111                 if (capability->sym.auth.algo == algo)
 112                         return 1;
 113         }
 114         return 0;
 115 }
 116
 117 /** Encrypt a single partial block
 118  *  Depends on openssl libcrypto
 119  *  Uses ECB+XOR to do CFB encryption, same result, more performant
 120  */
 121 static inline int
 122 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
 123                 uint8_t *iv, int ivlen, int srclen,
 124                 void *bpi_ctx)
 125 {
 126         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
 127         int encrypted_ivlen;
 128         uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
 129         uint8_t *encr = encrypted_iv;
 130
 131         /* ECB method: encrypt the IV, then XOR this with plaintext */
 132         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
 133                                                                 <= 0)
 134                 goto cipher_encrypt_err;
 135
 136         for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
 137                 *dst = *src ^ *encr;
 138
 139         return 0;
 140
 141 cipher_encrypt_err:
 142         PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
 143         return -EINVAL;
 144 }
 145
 146 /** Decrypt a single partial block
 147  *  Depends on openssl libcrypto
 148  *  Uses ECB+XOR to do CFB encryption, same result, more performant
 149  */
 150 static inline int
 151 bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
 152                 uint8_t *iv, int ivlen, int srclen,
 153                 void *bpi_ctx)
 154 {
 155         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
 156         int encrypted_ivlen;
 157         uint8_t encrypted_iv[BPI_MAX_ENCR_IV_LEN];
 158         uint8_t *encr = encrypted_iv;
 159
 160         /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
 161         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
 162                                                                 <= 0)
 163                 goto cipher_decrypt_err;
 164
 165         for (; srclen != 0; --srclen, ++dst, ++src, ++encr)
 166                 *dst = *src ^ *encr;
 167
 168         return 0;
 169
 170 cipher_decrypt_err:
 171         PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt for BPI IV failed");
 172         return -EINVAL;
 173 }
 174
 175 /** Creates a context in either AES or DES in ECB mode
 176  *  Depends on openssl libcrypto
 177  */
 178 static int
 179 bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
 180                 enum rte_crypto_cipher_operation direction __rte_unused,
 181                 uint8_t *key, void **ctx)
 182 {
 183         const EVP_CIPHER *algo = NULL;
 184         int ret;
 185         *ctx = EVP_CIPHER_CTX_new();
 186
 187         if (*ctx == NULL) {
 188                 ret = -ENOMEM;
 189                 goto ctx_init_err;
 190         }
 191
 192         if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
 193                 algo = EVP_des_ecb();
 194         else
 195                 algo = EVP_aes_128_ecb();
 196
 197         /* IV will be ECB encrypted whether direction is encrypt or decrypt*/
 198         if (EVP_EncryptInit_ex(*ctx, algo, NULL, key, 0) != 1) {
 199                 ret = -EINVAL;
 200                 goto ctx_init_err;
 201         }
 202
 203         return 0;
 204
 205 ctx_init_err:
 206         if (*ctx != NULL)
 207                 EVP_CIPHER_CTX_free(*ctx);
 208         return ret;
 209 }
 210
 211 /** Frees a context previously created
 212  *  Depends on openssl libcrypto
 213  */
 214 static void
 215 bpi_cipher_ctx_free(void *bpi_ctx)
 216 {
 217         if (bpi_ctx != NULL)
 218                 EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx);
 219 }
 220
 221 static inline uint32_t
 222 adf_modulo(uint32_t data, uint32_t shift);
 223
 224 static inline int
 225 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
 226                 struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp);
 227
 228 void
 229 qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 230                 struct rte_cryptodev_sym_session *sess)
 231 {
 232         PMD_INIT_FUNC_TRACE();
 233         uint8_t index = dev->driver_id;
 234         void *sess_priv = get_session_private_data(sess, index);
 235         struct qat_session *s = (struct qat_session *)sess_priv;
 236
 237         if (sess_priv) {
 238                 if (s->bpi_ctx)
 239                         bpi_cipher_ctx_free(s->bpi_ctx);
 240                 memset(s, 0, qat_crypto_sym_get_session_private_size(dev));
 241                 struct rte_mempool *sess_mp = rte_mempool_from_obj(sess_priv);
 242                 set_session_private_data(sess, index, NULL);
 243                 rte_mempool_put(sess_mp, sess_priv);
 244         }
 245 }
 246
 247 static int
 248 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 249 {
 250         /* Cipher Only */
 251         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 252                 return ICP_QAT_FW_LA_CMD_CIPHER;
 253
 254         /* Authentication Only */
 255         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 256                 return ICP_QAT_FW_LA_CMD_AUTH;
 257
 258         /* AEAD */
 259         if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 260                 /* AES-GCM and AES-CCM works with different direction
 261                  * GCM first encrypts and generate hash where AES-CCM
 262                  * first generate hash and encrypts. Similar relation
 263                  * applies to decryption.
 264                  */
 265                 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT)
 266                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 267                                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 268                         else
 269                                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 270                 else
 271                         if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_GCM)
 272                                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 273                         else
 274                                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 275         }
 276
 277         if (xform->next == NULL)
 278                 return -1;
 279
 280         /* Cipher then Authenticate */
 281         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 282                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 283                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 284
 285         /* Authenticate then Cipher */
 286         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 287                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 288                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 289
 290         return -1;
 291 }
 292
 293 static struct rte_crypto_auth_xform *
 294 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 295 {
 296         do {
 297                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 298                         return &xform->auth;
 299
 300                 xform = xform->next;
 301         } while (xform);
 302
 303         return NULL;
 304 }
 305
 306 static struct rte_crypto_cipher_xform *
 307 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 308 {
 309         do {
 310                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 311                         return &xform->cipher;
 312
 313                 xform = xform->next;
 314         } while (xform);
 315
 316         return NULL;
 317 }
 318
 319 int
 320 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 321                 struct rte_crypto_sym_xform *xform,
 322                 struct qat_session *session)
 323 {
 324         struct qat_pmd_private *internals = dev->data->dev_private;
 325         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 326         int ret;
 327
 328         /* Get cipher xform from crypto xform chain */
 329         cipher_xform = qat_get_cipher_xform(xform);
 330
 331         session->cipher_iv.offset = cipher_xform->iv.offset;
 332         session->cipher_iv.length = cipher_xform->iv.length;
 333
 334         switch (cipher_xform->algo) {
 335         case RTE_CRYPTO_CIPHER_AES_CBC:
 336                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 337                                 &session->qat_cipher_alg) != 0) {
 338                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 339                         ret = -EINVAL;
 340                         goto error_out;
 341                 }
 342                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 343                 break;
 344         case RTE_CRYPTO_CIPHER_AES_CTR:
 345                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 346                                 &session->qat_cipher_alg) != 0) {
 347                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 348                         ret = -EINVAL;
 349                         goto error_out;
 350                 }
 351                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 352                 break;
 353         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 354                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 355                                         &session->qat_cipher_alg) != 0) {
 356                         PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size");
 357                         ret = -EINVAL;
 358                         goto error_out;
 359                 }
 360                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 361                 break;
 362         case RTE_CRYPTO_CIPHER_NULL:
 363                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 364                 break;
 365         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 366                 if (qat_alg_validate_kasumi_key(cipher_xform->key.length,
 367                                         &session->qat_cipher_alg) != 0) {
 368                         PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size");
 369                         ret = -EINVAL;
 370                         goto error_out;
 371                 }
 372                 session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
 373                 break;
 374         case RTE_CRYPTO_CIPHER_3DES_CBC:
 375                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 376                                 &session->qat_cipher_alg) != 0) {
 377                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 378                         ret = -EINVAL;
 379                         goto error_out;
 380                 }
 381                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 382                 break;
 383         case RTE_CRYPTO_CIPHER_DES_CBC:
 384                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 385                                 &session->qat_cipher_alg) != 0) {
 386                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 387                         ret = -EINVAL;
 388                         goto error_out;
 389                 }
 390                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 391                 break;
 392         case RTE_CRYPTO_CIPHER_3DES_CTR:
 393                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 394                                 &session->qat_cipher_alg) != 0) {
 395                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 396                         ret = -EINVAL;
 397                         goto error_out;
 398                 }
 399                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 400                 break;
 401         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 402                 ret = bpi_cipher_ctx_init(
 403                                         cipher_xform->algo,
 404                                         cipher_xform->op,
 405                                         cipher_xform->key.data,
 406                                         &session->bpi_ctx);
 407                 if (ret != 0) {
 408                         PMD_DRV_LOG(ERR, "failed to create DES BPI ctx");
 409                         goto error_out;
 410                 }
 411                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 412                                 &session->qat_cipher_alg) != 0) {
 413                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 414                         ret = -EINVAL;
 415                         goto error_out;
 416                 }
 417                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 418                 break;
 419         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 420                 ret = bpi_cipher_ctx_init(
 421                                         cipher_xform->algo,
 422                                         cipher_xform->op,
 423                                         cipher_xform->key.data,
 424                                         &session->bpi_ctx);
 425                 if (ret != 0) {
 426                         PMD_DRV_LOG(ERR, "failed to create AES BPI ctx");
 427                         goto error_out;
 428                 }
 429                 if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length,
 430                                 &session->qat_cipher_alg) != 0) {
 431                         PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size");
 432                         ret = -EINVAL;
 433                         goto error_out;
 434                 }
 435                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 436                 break;
 437         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 438                 if (!qat_is_cipher_alg_supported(
 439                         cipher_xform->algo, internals)) {
 440                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 441                                 rte_crypto_cipher_algorithm_strings
 442                                         [cipher_xform->algo]);
 443                         ret = -ENOTSUP;
 444                         goto error_out;
 445                 }
 446                 if (qat_alg_validate_zuc_key(cipher_xform->key.length,
 447                                 &session->qat_cipher_alg) != 0) {
 448                         PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size");
 449                         ret = -EINVAL;
 450                         goto error_out;
 451                 }
 452                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 453                 break;
 454         case RTE_CRYPTO_CIPHER_3DES_ECB:
 455         case RTE_CRYPTO_CIPHER_AES_ECB:
 456         case RTE_CRYPTO_CIPHER_AES_F8:
 457         case RTE_CRYPTO_CIPHER_AES_XTS:
 458         case RTE_CRYPTO_CIPHER_ARC4:
 459                 PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u",
 460                                 cipher_xform->algo);
 461                 ret = -ENOTSUP;
 462                 goto error_out;
 463         default:
 464                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 465                                 cipher_xform->algo);
 466                 ret = -EINVAL;
 467                 goto error_out;
 468         }
 469
 470         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 471                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 472         else
 473                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 474
 475         if (qat_alg_aead_session_create_content_desc_cipher(session,
 476                                                 cipher_xform->key.data,
 477                                                 cipher_xform->key.length)) {
 478                 ret = -EINVAL;
 479                 goto error_out;
 480         }
 481
 482         return 0;
 483
 484 error_out:
 485         if (session->bpi_ctx) {
 486                 bpi_cipher_ctx_free(session->bpi_ctx);
 487                 session->bpi_ctx = NULL;
 488         }
 489         return ret;
 490 }
 491
 492 int
 493 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 494                 struct rte_crypto_sym_xform *xform,
 495                 struct rte_cryptodev_sym_session *sess,
 496                 struct rte_mempool *mempool)
 497 {
 498         void *sess_private_data;
 499         int ret;
 500
 501         if (rte_mempool_get(mempool, &sess_private_data)) {
 502                 CDEV_LOG_ERR(
 503                         "Couldn't get object from session mempool");
 504                 return -ENOMEM;
 505         }
 506
 507         ret = qat_crypto_set_session_parameters(dev, xform, sess_private_data);
 508         if (ret != 0) {
 509                 PMD_DRV_LOG(ERR, "Crypto QAT PMD: failed to configure "
 510                                 "session parameters");
 511
 512                 /* Return session to mempool */
 513                 rte_mempool_put(mempool, sess_private_data);
 514                 return ret;
 515         }
 516
 517         set_session_private_data(sess, dev->driver_id,
 518                 sess_private_data);
 519
 520         return 0;
 521 }
 522
 523 int
 524 qat_crypto_set_session_parameters(struct rte_cryptodev *dev,
 525                 struct rte_crypto_sym_xform *xform, void *session_private)
 526 {
 527         struct qat_session *session = session_private;
 528         int ret;
 529
 530         int qat_cmd_id;
 531         PMD_INIT_FUNC_TRACE();
 532
 533         /* Set context descriptor physical address */
 534         session->cd_paddr = rte_mempool_virt2iova(session) +
 535                         offsetof(struct qat_session, cd);
 536
 537         session->min_qat_dev_gen = QAT_GEN1;
 538
 539         /* Get requested QAT command id */
 540         qat_cmd_id = qat_get_cmd_id(xform);
 541         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 542                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 543                 return -ENOTSUP;
 544         }
 545         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 546         switch (session->qat_cmd) {
 547         case ICP_QAT_FW_LA_CMD_CIPHER:
 548                 ret = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 549                 if (ret < 0)
 550                         return ret;
 551                 break;
 552         case ICP_QAT_FW_LA_CMD_AUTH:
 553                 ret = qat_crypto_sym_configure_session_auth(dev, xform, session);
 554                 if (ret < 0)
 555                         return ret;
 556                 break;
 557         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 558                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 559                         ret = qat_crypto_sym_configure_session_aead(xform,
 560                                         session);
 561                         if (ret < 0)
 562                                 return ret;
 563                 } else {
 564                         ret = qat_crypto_sym_configure_session_cipher(dev,
 565                                         xform, session);
 566                         if (ret < 0)
 567                                 return ret;
 568                         ret = qat_crypto_sym_configure_session_auth(dev,
 569                                         xform, session);
 570                         if (ret < 0)
 571                                 return ret;
 572                 }
 573                 break;
 574         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 575                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) {
 576                         ret = qat_crypto_sym_configure_session_aead(xform,
 577                                         session);
 578                         if (ret < 0)
 579                                 return ret;
 580                 } else {
 581                         ret = qat_crypto_sym_configure_session_auth(dev,
 582                                         xform, session);
 583                         if (ret < 0)
 584                                 return ret;
 585                         ret = qat_crypto_sym_configure_session_cipher(dev,
 586                                         xform, session);
 587                         if (ret < 0)
 588                                 return ret;
 589                 }
 590                 break;
 591         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 592         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 593         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 594         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 595         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 596         case ICP_QAT_FW_LA_CMD_MGF1:
 597         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 598         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 599         case ICP_QAT_FW_LA_CMD_DELIMITER:
 600         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 601                 session->qat_cmd);
 602                 return -ENOTSUP;
 603         default:
 604         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 605                 session->qat_cmd);
 606                 return -ENOTSUP;
 607         }
 608
 609         return 0;
 610 }
 611
 612 int
 613 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 614                                 struct rte_crypto_sym_xform *xform,
 615                                 struct qat_session *session)
 616 {
 617         struct rte_crypto_auth_xform *auth_xform = NULL;
 618         struct qat_pmd_private *internals = dev->data->dev_private;
 619         auth_xform = qat_get_auth_xform(xform);
 620         uint8_t *key_data = auth_xform->key.data;
 621         uint8_t key_length = auth_xform->key.length;
 622
 623         switch (auth_xform->algo) {
 624         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 625                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 626                 break;
 627         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 628                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
 629                 break;
 630         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 631                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 632                 break;
 633         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 634                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
 635                 break;
 636         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 637                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 638                 break;
 639         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 640                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 641                 break;
 642         case RTE_CRYPTO_AUTH_AES_GMAC:
 643                 if (qat_alg_validate_aes_key(auth_xform->key.length,
 644                                 &session->qat_cipher_alg) != 0) {
 645                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 646                         return -EINVAL;
 647                 }
 648                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 649                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 650
 651                 break;
 652         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 653                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 654                 break;
 655         case RTE_CRYPTO_AUTH_MD5_HMAC:
 656                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
 657                 break;
 658         case RTE_CRYPTO_AUTH_NULL:
 659                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
 660                 break;
 661         case RTE_CRYPTO_AUTH_KASUMI_F9:
 662                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
 663                 break;
 664         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 665                 if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) {
 666                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 667                                 rte_crypto_auth_algorithm_strings
 668                                 [auth_xform->algo]);
 669                         return -ENOTSUP;
 670                 }
 671                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
 672                 break;
 673         case RTE_CRYPTO_AUTH_SHA1:
 674         case RTE_CRYPTO_AUTH_SHA256:
 675         case RTE_CRYPTO_AUTH_SHA512:
 676         case RTE_CRYPTO_AUTH_SHA224:
 677         case RTE_CRYPTO_AUTH_SHA384:
 678         case RTE_CRYPTO_AUTH_MD5:
 679         case RTE_CRYPTO_AUTH_AES_CMAC:
 680         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 681                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 682                                 auth_xform->algo);
 683                 return -ENOTSUP;
 684         default:
 685                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 686                                 auth_xform->algo);
 687                 return -EINVAL;
 688         }
 689
 690         session->auth_iv.offset = auth_xform->iv.offset;
 691         session->auth_iv.length = auth_xform->iv.length;
 692
 693         if (auth_xform->algo == RTE_CRYPTO_AUTH_AES_GMAC) {
 694                 if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) {
 695                         session->qat_cmd = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 696                         session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 697                         /*
 698                          * It needs to create cipher desc content first,
 699                          * then authentication
 700                          */
 701                         if (qat_alg_aead_session_create_content_desc_cipher(session,
 702                                                 auth_xform->key.data,
 703                                                 auth_xform->key.length))
 704                                 return -EINVAL;
 705
 706                         if (qat_alg_aead_session_create_content_desc_auth(session,
 707                                                 key_data,
 708                                                 key_length,
 709                                                 0,
 710                                                 auth_xform->digest_length,
 711                                                 auth_xform->op))
 712                                 return -EINVAL;
 713                 } else {
 714                         session->qat_cmd = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 715                         session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 716                         /*
 717                          * It needs to create authentication desc content first,
 718                          * then cipher
 719                          */
 720                         if (qat_alg_aead_session_create_content_desc_auth(session,
 721                                         key_data,
 722                                         key_length,
 723                                         0,
 724                                         auth_xform->digest_length,
 725                                         auth_xform->op))
 726                                 return -EINVAL;
 727
 728                         if (qat_alg_aead_session_create_content_desc_cipher(session,
 729                                                 auth_xform->key.data,
 730                                                 auth_xform->key.length))
 731                                 return -EINVAL;
 732                 }
 733                 /* Restore to authentication only only */
 734                 session->qat_cmd = ICP_QAT_FW_LA_CMD_AUTH;
 735         } else {
 736                 if (qat_alg_aead_session_create_content_desc_auth(session,
 737                                 key_data,
 738                                 key_length,
 739                                 0,
 740                                 auth_xform->digest_length,
 741                                 auth_xform->op))
 742                         return -EINVAL;
 743         }
 744
 745         session->digest_length = auth_xform->digest_length;
 746         return 0;
 747 }
 748
 749 int
 750 qat_crypto_sym_configure_session_aead(struct rte_crypto_sym_xform *xform,
 751                                 struct qat_session *session)
 752 {
 753         struct rte_crypto_aead_xform *aead_xform = &xform->aead;
 754         enum rte_crypto_auth_operation crypto_operation;
 755
 756         /*
 757          * Store AEAD IV parameters as cipher IV,
 758          * to avoid unnecessary memory usage
 759          */
 760         session->cipher_iv.offset = xform->aead.iv.offset;
 761         session->cipher_iv.length = xform->aead.iv.length;
 762
 763         switch (aead_xform->algo) {
 764         case RTE_CRYPTO_AEAD_AES_GCM:
 765                 if (qat_alg_validate_aes_key(aead_xform->key.length,
 766                                 &session->qat_cipher_alg) != 0) {
 767                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 768                         return -EINVAL;
 769                 }
 770                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 771                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 772                 break;
 773         case RTE_CRYPTO_AEAD_AES_CCM:
 774                 if (qat_alg_validate_aes_key(aead_xform->key.length,
 775                                 &session->qat_cipher_alg) != 0) {
 776                         PMD_DRV_LOG(ERR, "Invalid AES key size");
 777                         return -EINVAL;
 778                 }
 779                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 780                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC;
 781                 break;
 782         default:
 783                 PMD_DRV_LOG(ERR, "Crypto: Undefined AEAD specified %u\n",
 784                                 aead_xform->algo);
 785                 return -EINVAL;
 786         }
 787
 788         if ((aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT &&
 789                         aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM) ||
 790                         (aead_xform->op == RTE_CRYPTO_AEAD_OP_DECRYPT &&
 791                         aead_xform->algo == RTE_CRYPTO_AEAD_AES_CCM)) {
 792                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 793                 /*
 794                  * It needs to create cipher desc content first,
 795                  * then authentication
 796                  */
 797
 798                 crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
 799                         RTE_CRYPTO_AUTH_OP_GENERATE : RTE_CRYPTO_AUTH_OP_VERIFY;
 800
 801                 if (qat_alg_aead_session_create_content_desc_cipher(session,
 802                                         aead_xform->key.data,
 803                                         aead_xform->key.length))
 804                         return -EINVAL;
 805
 806                 if (qat_alg_aead_session_create_content_desc_auth(session,
 807                                         aead_xform->key.data,
 808                                         aead_xform->key.length,
 809                                         aead_xform->aad_length,
 810                                         aead_xform->digest_length,
 811                                         crypto_operation))
 812                         return -EINVAL;
 813         } else {
 814                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 815                 /*
 816                  * It needs to create authentication desc content first,
 817                  * then cipher
 818                  */
 819
 820                 crypto_operation = aead_xform->algo == RTE_CRYPTO_AEAD_AES_GCM ?
 821                         RTE_CRYPTO_AUTH_OP_VERIFY : RTE_CRYPTO_AUTH_OP_GENERATE;
 822
 823                 if (qat_alg_aead_session_create_content_desc_auth(session,
 824                                         aead_xform->key.data,
 825                                         aead_xform->key.length,
 826                                         aead_xform->aad_length,
 827                                         aead_xform->digest_length,
 828                                         crypto_operation))
 829                         return -EINVAL;
 830
 831                 if (qat_alg_aead_session_create_content_desc_cipher(session,
 832                                         aead_xform->key.data,
 833                                         aead_xform->key.length))
 834                         return -EINVAL;
 835         }
 836
 837         session->digest_length = aead_xform->digest_length;
 838         return 0;
 839 }
 840
 841 unsigned qat_crypto_sym_get_session_private_size(
 842                 struct rte_cryptodev *dev __rte_unused)
 843 {
 844         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 845 }
 846
 847 static inline uint32_t
 848 qat_bpicipher_preprocess(struct qat_session *ctx,
 849                                 struct rte_crypto_op *op)
 850 {
 851         int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
 852         struct rte_crypto_sym_op *sym_op = op->sym;
 853         uint8_t last_block_len = block_len > 0 ?
 854                         sym_op->cipher.data.length % block_len : 0;
 855
 856         if (last_block_len &&
 857                         ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
 858
 859                 /* Decrypt last block */
 860                 uint8_t *last_block, *dst, *iv;
 861                 uint32_t last_block_offset = sym_op->cipher.data.offset +
 862                                 sym_op->cipher.data.length - last_block_len;
 863                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
 864                                 uint8_t *, last_block_offset);
 865
 866                 if (unlikely(sym_op->m_dst != NULL))
 867                         /* out-of-place operation (OOP) */
 868                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
 869                                                 uint8_t *, last_block_offset);
 870                 else
 871                         dst = last_block;
 872
 873                 if (last_block_len < sym_op->cipher.data.length)
 874                         /* use previous block ciphertext as IV */
 875                         iv = last_block - block_len;
 876                 else
 877                         /* runt block, i.e. less than one full block */
 878                         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 879                                         ctx->cipher_iv.offset);
 880
 881 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 882                 rte_hexdump(stdout, "BPI: src before pre-process:", last_block,
 883                         last_block_len);
 884                 if (sym_op->m_dst != NULL)
 885                         rte_hexdump(stdout, "BPI: dst before pre-process:", dst,
 886                                 last_block_len);
 887 #endif
 888                 bpi_cipher_decrypt(last_block, dst, iv, block_len,
 889                                 last_block_len, ctx->bpi_ctx);
 890 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 891                 rte_hexdump(stdout, "BPI: src after pre-process:", last_block,
 892                         last_block_len);
 893                 if (sym_op->m_dst != NULL)
 894                         rte_hexdump(stdout, "BPI: dst after pre-process:", dst,
 895                                 last_block_len);
 896 #endif
 897         }
 898
 899         return sym_op->cipher.data.length - last_block_len;
 900 }
 901
 902 static inline uint32_t
 903 qat_bpicipher_postprocess(struct qat_session *ctx,
 904                                 struct rte_crypto_op *op)
 905 {
 906         int block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
 907         struct rte_crypto_sym_op *sym_op = op->sym;
 908         uint8_t last_block_len = block_len > 0 ?
 909                         sym_op->cipher.data.length % block_len : 0;
 910
 911         if (last_block_len > 0 &&
 912                         ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
 913
 914                 /* Encrypt last block */
 915                 uint8_t *last_block, *dst, *iv;
 916                 uint32_t last_block_offset;
 917
 918                 last_block_offset = sym_op->cipher.data.offset +
 919                                 sym_op->cipher.data.length - last_block_len;
 920                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
 921                                 uint8_t *, last_block_offset);
 922
 923                 if (unlikely(sym_op->m_dst != NULL))
 924                         /* out-of-place operation (OOP) */
 925                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
 926                                                 uint8_t *, last_block_offset);
 927                 else
 928                         dst = last_block;
 929
 930                 if (last_block_len < sym_op->cipher.data.length)
 931                         /* use previous block ciphertext as IV */
 932                         iv = dst - block_len;
 933                 else
 934                         /* runt block, i.e. less than one full block */
 935                         iv = rte_crypto_op_ctod_offset(op, uint8_t *,
 936                                         ctx->cipher_iv.offset);
 937
 938 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 939                 rte_hexdump(stdout, "BPI: src before post-process:", last_block,
 940                         last_block_len);
 941                 if (sym_op->m_dst != NULL)
 942                         rte_hexdump(stdout, "BPI: dst before post-process:",
 943                                         dst, last_block_len);
 944 #endif
 945                 bpi_cipher_encrypt(last_block, dst, iv, block_len,
 946                                 last_block_len, ctx->bpi_ctx);
 947 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 948                 rte_hexdump(stdout, "BPI: src after post-process:", last_block,
 949                         last_block_len);
 950                 if (sym_op->m_dst != NULL)
 951                         rte_hexdump(stdout, "BPI: dst after post-process:", dst,
 952                                 last_block_len);
 953 #endif
 954         }
 955         return sym_op->cipher.data.length - last_block_len;
 956 }
 957
 958 static inline void
 959 txq_write_tail(struct qat_qp *qp, struct qat_queue *q) {
 960         WRITE_CSR_RING_TAIL(qp->mmap_bar_addr, q->hw_bundle_number,
 961                         q->hw_queue_number, q->tail);
 962         q->nb_pending_requests = 0;
 963         q->csr_tail = q->tail;
 964 }
 965
 966 uint16_t
 967 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 968                 uint16_t nb_ops)
 969 {
 970         register struct qat_queue *queue;
 971         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 972         register uint32_t nb_ops_sent = 0;
 973         register struct rte_crypto_op **cur_op = ops;
 974         register int ret;
 975         uint16_t nb_ops_possible = nb_ops;
 976         register uint8_t *base_addr;
 977         register uint32_t tail;
 978         int overflow;
 979
 980         if (unlikely(nb_ops == 0))
 981                 return 0;
 982
 983         /* read params used a lot in main loop into registers */
 984         queue = &(tmp_qp->tx_q);
 985         base_addr = (uint8_t *)queue->base_addr;
 986         tail = queue->tail;
 987
 988         /* Find how many can actually fit on the ring */
 989         tmp_qp->inflights16 += nb_ops;
 990         overflow = tmp_qp->inflights16 - queue->max_inflights;
 991         if (overflow > 0) {
 992                 tmp_qp->inflights16 -= overflow;
 993                 nb_ops_possible = nb_ops - overflow;
 994                 if (nb_ops_possible == 0)
 995                         return 0;
 996         }
 997
 998         while (nb_ops_sent != nb_ops_possible) {
 999                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail,
1000                         tmp_qp->op_cookies[tail / queue->msg_size], tmp_qp);
1001                 if (ret != 0) {
1002                         tmp_qp->stats.enqueue_err_count++;
1003                         /*
1004                          * This message cannot be enqueued,
1005                          * decrease number of ops that wasn't sent
1006                          */
1007                         tmp_qp->inflights16 -= nb_ops_possible - nb_ops_sent;
1008                         if (nb_ops_sent == 0)
1009                                 return 0;
1010                         goto kick_tail;
1011                 }
1012
1013                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
1014                 nb_ops_sent++;
1015                 cur_op++;
1016         }
1017 kick_tail:
1018         queue->tail = tail;
1019         tmp_qp->stats.enqueued_count += nb_ops_sent;
1020         queue->nb_pending_requests += nb_ops_sent;
1021         if (tmp_qp->inflights16 < QAT_CSR_TAIL_FORCE_WRITE_THRESH ||
1022                         queue->nb_pending_requests > QAT_CSR_TAIL_WRITE_THRESH) {
1023                 txq_write_tail(tmp_qp, queue);
1024         }
1025         return nb_ops_sent;
1026 }
1027
1028 static inline
1029 void rxq_free_desc(struct qat_qp *qp, struct qat_queue *q)
1030 {
1031         uint32_t old_head, new_head;
1032         uint32_t max_head;
1033
1034         old_head = q->csr_head;
1035         new_head = q->head;
1036         max_head = qp->nb_descriptors * q->msg_size;
1037
1038         /* write out free descriptors */
1039         void *cur_desc = (uint8_t *)q->base_addr + old_head;
1040
1041         if (new_head < old_head) {
1042                 memset(cur_desc, ADF_RING_EMPTY_SIG, max_head - old_head);
1043                 memset(q->base_addr, ADF_RING_EMPTY_SIG, new_head);
1044         } else {
1045                 memset(cur_desc, ADF_RING_EMPTY_SIG, new_head - old_head);
1046         }
1047         q->nb_processed_responses = 0;
1048         q->csr_head = new_head;
1049
1050         /* write current head to CSR */
1051         WRITE_CSR_RING_HEAD(qp->mmap_bar_addr, q->hw_bundle_number,
1052                             q->hw_queue_number, new_head);
1053 }
1054
1055 uint16_t
1056 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
1057                 uint16_t nb_ops)
1058 {
1059         struct qat_queue *rx_queue, *tx_queue;
1060         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
1061         uint32_t msg_counter = 0;
1062         struct rte_crypto_op *rx_op;
1063         struct icp_qat_fw_comn_resp *resp_msg;
1064         uint32_t head;
1065
1066         rx_queue = &(tmp_qp->rx_q);
1067         tx_queue = &(tmp_qp->tx_q);
1068         head = rx_queue->head;
1069         resp_msg = (struct icp_qat_fw_comn_resp *)
1070                         ((uint8_t *)rx_queue->base_addr + head);
1071
1072         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
1073                         msg_counter != nb_ops) {
1074                 rx_op = (struct rte_crypto_op *)(uintptr_t)
1075                                 (resp_msg->opaque_data);
1076
1077 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
1078                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
1079                         sizeof(struct icp_qat_fw_comn_resp));
1080 #endif
1081                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
1082                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
1083                                         resp_msg->comn_hdr.comn_status)) {
1084                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
1085                 } else {
1086                         struct qat_session *sess = (struct qat_session *)
1087                                         get_session_private_data(
1088                                         rx_op->sym->session,
1089                                         cryptodev_qat_driver_id);
1090
1091                         if (sess->bpi_ctx)
1092                                 qat_bpicipher_postprocess(sess, rx_op);
1093                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
1094                 }
1095
1096                 head = adf_modulo(head + rx_queue->msg_size, rx_queue->modulo);
1097                 resp_msg = (struct icp_qat_fw_comn_resp *)
1098                                 ((uint8_t *)rx_queue->base_addr + head);
1099                 *ops = rx_op;
1100                 ops++;
1101                 msg_counter++;
1102         }
1103         if (msg_counter > 0) {
1104                 rx_queue->head = head;
1105                 tmp_qp->stats.dequeued_count += msg_counter;
1106                 rx_queue->nb_processed_responses += msg_counter;
1107                 tmp_qp->inflights16 -= msg_counter;
1108
1109                 if (rx_queue->nb_processed_responses > QAT_CSR_HEAD_WRITE_THRESH)
1110                         rxq_free_desc(tmp_qp, rx_queue);
1111         }
1112         /* also check if tail needs to be advanced */
1113         if (tmp_qp->inflights16 <= QAT_CSR_TAIL_FORCE_WRITE_THRESH &&
1114                         tx_queue->tail != tx_queue->csr_tail) {
1115                 txq_write_tail(tmp_qp, tx_queue);
1116         }
1117         return msg_counter;
1118 }
1119
1120 static inline int
1121 qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start,
1122                 struct qat_alg_buf_list *list, uint32_t data_len)
1123 {
1124         int nr = 1;
1125
1126         uint32_t buf_len = rte_pktmbuf_iova(buf) -
1127                         buff_start + rte_pktmbuf_data_len(buf);
1128
1129         list->bufers[0].addr = buff_start;
1130         list->bufers[0].resrvd = 0;
1131         list->bufers[0].len = buf_len;
1132
1133         if (data_len <= buf_len) {
1134                 list->num_bufs = nr;
1135                 list->bufers[0].len = data_len;
1136                 return 0;
1137         }
1138
1139         buf = buf->next;
1140         while (buf) {
1141                 if (unlikely(nr == QAT_SGL_MAX_NUMBER)) {
1142                         PMD_DRV_LOG(ERR, "QAT PMD exceeded size of QAT SGL"
1143                                         " entry(%u)",
1144                                         QAT_SGL_MAX_NUMBER);
1145                         return -EINVAL;
1146                 }
1147
1148                 list->bufers[nr].len = rte_pktmbuf_data_len(buf);
1149                 list->bufers[nr].resrvd = 0;
1150                 list->bufers[nr].addr = rte_pktmbuf_iova(buf);
1151
1152                 buf_len += list->bufers[nr].len;
1153                 buf = buf->next;
1154
1155                 if (buf_len > data_len) {
1156                         list->bufers[nr].len -=
1157                                 buf_len - data_len;
1158                         buf = NULL;
1159                 }
1160                 ++nr;
1161         }
1162         list->num_bufs = nr;
1163
1164         return 0;
1165 }
1166
1167 static inline void
1168 set_cipher_iv(uint16_t iv_length, uint16_t iv_offset,
1169                 struct icp_qat_fw_la_cipher_req_params *cipher_param,
1170                 struct rte_crypto_op *op,
1171                 struct icp_qat_fw_la_bulk_req *qat_req)
1172 {
1173         /* copy IV into request if it fits */
1174         if (iv_length <= sizeof(cipher_param->u.cipher_IV_array)) {
1175                 rte_memcpy(cipher_param->u.cipher_IV_array,
1176                                 rte_crypto_op_ctod_offset(op, uint8_t *,
1177                                         iv_offset),
1178                                 iv_length);
1179         } else {
1180                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
1181                                 qat_req->comn_hdr.serv_specif_flags,
1182                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
1183                 cipher_param->u.s.cipher_IV_ptr =
1184                                 rte_crypto_op_ctophys_offset(op,
1185                                         iv_offset);
1186         }
1187 }
1188
1189 /** Set IV for CCM is special case, 0th byte is set to q-1
1190  *  where q is padding of nonce in 16 byte block
1191  */
1192 static inline void
1193 set_cipher_iv_ccm(uint16_t iv_length, uint16_t iv_offset,
1194                 struct icp_qat_fw_la_cipher_req_params *cipher_param,
1195                 struct rte_crypto_op *op, uint8_t q, uint8_t aad_len_field_sz)
1196 {
1197         rte_memcpy(((uint8_t *)cipher_param->u.cipher_IV_array) +
1198                         ICP_QAT_HW_CCM_NONCE_OFFSET,
1199                         rte_crypto_op_ctod_offset(op, uint8_t *,
1200                                 iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
1201                         iv_length);
1202         *(uint8_t *)&cipher_param->u.cipher_IV_array[0] =
1203                         q - ICP_QAT_HW_CCM_NONCE_OFFSET;
1204
1205         if (aad_len_field_sz)
1206                 rte_memcpy(&op->sym->aead.aad.data[ICP_QAT_HW_CCM_NONCE_OFFSET],
1207                         rte_crypto_op_ctod_offset(op, uint8_t *,
1208                                 iv_offset) + ICP_QAT_HW_CCM_NONCE_OFFSET,
1209                         iv_length);
1210 }
1211
1212 static inline int
1213 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
1214                 struct qat_crypto_op_cookie *qat_op_cookie, struct qat_qp *qp)
1215 {
1216         int ret = 0;
1217         struct qat_session *ctx;
1218         struct icp_qat_fw_la_cipher_req_params *cipher_param;
1219         struct icp_qat_fw_la_auth_req_params *auth_param;
1220         register struct icp_qat_fw_la_bulk_req *qat_req;
1221         uint8_t do_auth = 0, do_cipher = 0, do_aead = 0;
1222         uint32_t cipher_len = 0, cipher_ofs = 0;
1223         uint32_t auth_len = 0, auth_ofs = 0;
1224         uint32_t min_ofs = 0;
1225         uint64_t src_buf_start = 0, dst_buf_start = 0;
1226         uint8_t do_sgl = 0;
1227
1228 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
1229         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
1230                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
1231                                 "operation requests, op (%p) is not a "
1232                                 "symmetric operation.", op);
1233                 return -EINVAL;
1234         }
1235 #endif
1236         if (unlikely(op->sess_type == RTE_CRYPTO_OP_SESSIONLESS)) {
1237                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
1238                                 " requests, op (%p) is sessionless.", op);
1239                 return -EINVAL;
1240         }
1241
1242         ctx = (struct qat_session *)get_session_private_data(
1243                         op->sym->session, cryptodev_qat_driver_id);
1244
1245         if (unlikely(ctx == NULL)) {
1246                 PMD_DRV_LOG(ERR, "Session was not created for this device");
1247                 return -EINVAL;
1248         }
1249
1250         if (unlikely(ctx->min_qat_dev_gen > qp->qat_dev_gen)) {
1251                 PMD_DRV_LOG(ERR, "Session alg not supported on this device gen");
1252                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
1253                 return -EINVAL;
1254         }
1255
1256
1257
1258         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
1259         rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
1260         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
1261         cipher_param = (void *)&qat_req->serv_specif_rqpars;
1262         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
1263
1264         if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
1265                         ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
1266                 /* AES-GCM or AES-CCM */
1267                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1268                                 ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64 ||
1269                                 (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_AES128
1270                                 && ctx->qat_mode == ICP_QAT_HW_CIPHER_CTR_MODE
1271                                 && ctx->qat_hash_alg ==
1272                                                 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC)) {
1273                         do_aead = 1;
1274                 } else {
1275                         do_auth = 1;
1276                         do_cipher = 1;
1277                 }
1278         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
1279                 do_auth = 1;
1280                 do_cipher = 0;
1281         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
1282                 do_auth = 0;
1283                 do_cipher = 1;
1284         }
1285
1286         if (do_cipher) {
1287
1288                 if (ctx->qat_cipher_alg ==
1289                                          ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
1290                         ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
1291                         ctx->qat_cipher_alg ==
1292                                 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
1293
1294                         if (unlikely(
1295                             (op->sym->cipher.data.length % BYTE_LENGTH != 0) ||
1296                             (op->sym->cipher.data.offset % BYTE_LENGTH != 0))) {
1297                                 PMD_DRV_LOG(ERR,
1298                   "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
1299                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1300                                 return -EINVAL;
1301                         }
1302                         cipher_len = op->sym->cipher.data.length >> 3;
1303                         cipher_ofs = op->sym->cipher.data.offset >> 3;
1304
1305                 } else if (ctx->bpi_ctx) {
1306                         /* DOCSIS - only send complete blocks to device
1307                          * Process any partial block using CFB mode.
1308                          * Even if 0 complete blocks, still send this to device
1309                          * to get into rx queue for post-process and dequeuing
1310                          */
1311                         cipher_len = qat_bpicipher_preprocess(ctx, op);
1312                         cipher_ofs = op->sym->cipher.data.offset;
1313                 } else {
1314                         cipher_len = op->sym->cipher.data.length;
1315                         cipher_ofs = op->sym->cipher.data.offset;
1316                 }
1317
1318                 set_cipher_iv(ctx->cipher_iv.length, ctx->cipher_iv.offset,
1319                                 cipher_param, op, qat_req);
1320                 min_ofs = cipher_ofs;
1321         }
1322
1323         if (do_auth) {
1324
1325                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
1326                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
1327                         ctx->qat_hash_alg ==
1328                                 ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
1329                         if (unlikely(
1330                             (op->sym->auth.data.offset % BYTE_LENGTH != 0) ||
1331                             (op->sym->auth.data.length % BYTE_LENGTH != 0))) {
1332                                 PMD_DRV_LOG(ERR,
1333                 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
1334                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1335                                 return -EINVAL;
1336                         }
1337                         auth_ofs = op->sym->auth.data.offset >> 3;
1338                         auth_len = op->sym->auth.data.length >> 3;
1339
1340                         auth_param->u1.aad_adr =
1341                                         rte_crypto_op_ctophys_offset(op,
1342                                                         ctx->auth_iv.offset);
1343
1344                 } else if (ctx->qat_hash_alg ==
1345                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1346                                 ctx->qat_hash_alg ==
1347                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
1348                         /* AES-GMAC */
1349                         set_cipher_iv(ctx->auth_iv.length,
1350                                 ctx->auth_iv.offset,
1351                                 cipher_param, op, qat_req);
1352                         auth_ofs = op->sym->auth.data.offset;
1353                         auth_len = op->sym->auth.data.length;
1354
1355                         auth_param->u1.aad_adr = 0;
1356                         auth_param->u2.aad_sz = 0;
1357
1358                         /*
1359                          * If len(iv)==12B fw computes J0
1360                          */
1361                         if (ctx->auth_iv.length == 12) {
1362                                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1363                                         qat_req->comn_hdr.serv_specif_flags,
1364                                         ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
1365
1366                         }
1367                 } else {
1368                         auth_ofs = op->sym->auth.data.offset;
1369                         auth_len = op->sym->auth.data.length;
1370
1371                 }
1372                 min_ofs = auth_ofs;
1373
1374                 if (likely(ctx->qat_hash_alg != ICP_QAT_HW_AUTH_ALGO_NULL))
1375                         auth_param->auth_res_addr =
1376                                         op->sym->auth.digest.phys_addr;
1377
1378         }
1379
1380         if (do_aead) {
1381                 /*
1382                  * This address may used for setting AAD physical pointer
1383                  * into IV offset from op
1384                  */
1385                 rte_iova_t aad_phys_addr_aead = op->sym->aead.aad.phys_addr;
1386                 if (ctx->qat_hash_alg ==
1387                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1388                                 ctx->qat_hash_alg ==
1389                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
1390                         /*
1391                          * If len(iv)==12B fw computes J0
1392                          */
1393                         if (ctx->cipher_iv.length == 12) {
1394                                 ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1395                                         qat_req->comn_hdr.serv_specif_flags,
1396                                         ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
1397                         }
1398
1399                         set_cipher_iv(ctx->cipher_iv.length,
1400                                         ctx->cipher_iv.offset,
1401                                         cipher_param, op, qat_req);
1402
1403                 } else if (ctx->qat_hash_alg ==
1404                                 ICP_QAT_HW_AUTH_ALGO_AES_CBC_MAC) {
1405
1406                         /* In case of AES-CCM this may point to user selected memory
1407                          * or iv offset in cypto_op
1408                          */
1409                         uint8_t *aad_data = op->sym->aead.aad.data;
1410                         /* This is true AAD length, it not includes 18 bytes of
1411                          * preceding data
1412                          */
1413                         uint8_t aad_ccm_real_len = 0;
1414
1415                         uint8_t aad_len_field_sz = 0;
1416                         uint32_t msg_len_be =
1417                                         rte_bswap32(op->sym->aead.data.length);
1418
1419                         if (ctx->aad_len > ICP_QAT_HW_CCM_AAD_DATA_OFFSET) {
1420                                 aad_len_field_sz = ICP_QAT_HW_CCM_AAD_LEN_INFO;
1421                                 aad_ccm_real_len = ctx->aad_len -
1422                                         ICP_QAT_HW_CCM_AAD_B0_LEN -
1423                                         ICP_QAT_HW_CCM_AAD_LEN_INFO;
1424                         } else {
1425                                 /*
1426                                  * aad_len not greater than 18, so no actual aad data,
1427                                  * then use IV after op for B0 block
1428                                  */
1429                                 aad_data = rte_crypto_op_ctod_offset(op, uint8_t *,
1430                                                 ctx->cipher_iv.offset);
1431                                 aad_phys_addr_aead =
1432                                                 rte_crypto_op_ctophys_offset(op,
1433                                                                 ctx->cipher_iv.offset);
1434                         }
1435
1436                         uint8_t q = ICP_QAT_HW_CCM_NQ_CONST - ctx->cipher_iv.length;
1437
1438                         aad_data[0] = ICP_QAT_HW_CCM_BUILD_B0_FLAGS(aad_len_field_sz,
1439                                                         ctx->digest_length, q);
1440
1441                         if (q > ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE) {
1442                                 memcpy(aad_data + ctx->cipher_iv.length +
1443                                         ICP_QAT_HW_CCM_NONCE_OFFSET
1444                                         + (q - ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE),
1445                                         (uint8_t *)&msg_len_be,
1446                                         ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE);
1447                         } else {
1448                                 memcpy(aad_data + ctx->cipher_iv.length +
1449                                         ICP_QAT_HW_CCM_NONCE_OFFSET,
1450                                         (uint8_t *)&msg_len_be
1451                                         + (ICP_QAT_HW_CCM_MSG_LEN_MAX_FIELD_SIZE
1452                                         - q), q);
1453                         }
1454
1455                         if (aad_len_field_sz > 0) {
1456                                 *(uint16_t *)&aad_data[ICP_QAT_HW_CCM_AAD_B0_LEN]
1457                                                 = rte_bswap16(aad_ccm_real_len);
1458
1459                                 if ((aad_ccm_real_len + aad_len_field_sz)
1460                                                 % ICP_QAT_HW_CCM_AAD_B0_LEN) {
1461                                         uint8_t pad_len = 0;
1462                                         uint8_t pad_idx = 0;
1463
1464                                         pad_len = ICP_QAT_HW_CCM_AAD_B0_LEN -
1465                                                 ((aad_ccm_real_len + aad_len_field_sz) %
1466                                                         ICP_QAT_HW_CCM_AAD_B0_LEN);
1467                                         pad_idx = ICP_QAT_HW_CCM_AAD_B0_LEN +
1468                                                 aad_ccm_real_len + aad_len_field_sz;
1469                                         memset(&aad_data[pad_idx],
1470                                                         0, pad_len);
1471                                 }
1472
1473                         }
1474
1475                         set_cipher_iv_ccm(ctx->cipher_iv.length,
1476                                         ctx->cipher_iv.offset,
1477                                         cipher_param, op, q,
1478                                         aad_len_field_sz);
1479
1480                 }
1481
1482                 cipher_len = op->sym->aead.data.length;
1483                 cipher_ofs = op->sym->aead.data.offset;
1484                 auth_len = op->sym->aead.data.length;
1485                 auth_ofs = op->sym->aead.data.offset;
1486
1487                 auth_param->u1.aad_adr = aad_phys_addr_aead;
1488                 auth_param->auth_res_addr = op->sym->aead.digest.phys_addr;
1489                 min_ofs = op->sym->aead.data.offset;
1490         }
1491
1492         if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next))
1493                 do_sgl = 1;
1494
1495         /* adjust for chain case */
1496         if (do_cipher && do_auth)
1497                 min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
1498
1499         if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl))
1500                 min_ofs = 0;
1501
1502         if (unlikely(op->sym->m_dst != NULL)) {
1503                 /* Out-of-place operation (OOP)
1504                  * Don't align DMA start. DMA the minimum data-set
1505                  * so as not to overwrite data in dest buffer
1506                  */
1507                 src_buf_start =
1508                         rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs);
1509                 dst_buf_start =
1510                         rte_pktmbuf_iova_offset(op->sym->m_dst, min_ofs);
1511
1512         } else {
1513                 /* In-place operation
1514                  * Start DMA at nearest aligned address below min_ofs
1515                  */
1516                 src_buf_start =
1517                         rte_pktmbuf_iova_offset(op->sym->m_src, min_ofs)
1518                                                 & QAT_64_BTYE_ALIGN_MASK;
1519
1520                 if (unlikely((rte_pktmbuf_iova(op->sym->m_src) -
1521                                         rte_pktmbuf_headroom(op->sym->m_src))
1522                                                         > src_buf_start)) {
1523                         /* alignment has pushed addr ahead of start of mbuf
1524                          * so revert and take the performance hit
1525                          */
1526                         src_buf_start =
1527                                 rte_pktmbuf_iova_offset(op->sym->m_src,
1528                                                                 min_ofs);
1529                 }
1530                 dst_buf_start = src_buf_start;
1531         }
1532
1533         if (do_cipher || do_aead) {
1534                 cipher_param->cipher_offset =
1535                                 (uint32_t)rte_pktmbuf_iova_offset(
1536                                 op->sym->m_src, cipher_ofs) - src_buf_start;
1537                 cipher_param->cipher_length = cipher_len;
1538         } else {
1539                 cipher_param->cipher_offset = 0;
1540                 cipher_param->cipher_length = 0;
1541         }
1542
1543         if (do_auth || do_aead) {
1544                 auth_param->auth_off = (uint32_t)rte_pktmbuf_iova_offset(
1545                                 op->sym->m_src, auth_ofs) - src_buf_start;
1546                 auth_param->auth_len = auth_len;
1547         } else {
1548                 auth_param->auth_off = 0;
1549                 auth_param->auth_len = 0;
1550         }
1551
1552         qat_req->comn_mid.dst_length =
1553                 qat_req->comn_mid.src_length =
1554                 (cipher_param->cipher_offset + cipher_param->cipher_length)
1555                 > (auth_param->auth_off + auth_param->auth_len) ?
1556                 (cipher_param->cipher_offset + cipher_param->cipher_length)
1557                 : (auth_param->auth_off + auth_param->auth_len);
1558
1559         if (do_sgl) {
1560
1561                 ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags,
1562                                 QAT_COMN_PTR_TYPE_SGL);
1563                 ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start,
1564                                 &qat_op_cookie->qat_sgl_list_src,
1565                                 qat_req->comn_mid.src_length);
1566                 if (ret) {
1567                         PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array");
1568                         return ret;
1569                 }
1570
1571                 if (likely(op->sym->m_dst == NULL))
1572                         qat_req->comn_mid.dest_data_addr =
1573                                 qat_req->comn_mid.src_data_addr =
1574                                 qat_op_cookie->qat_sgl_src_phys_addr;
1575                 else {
1576                         ret = qat_sgl_fill_array(op->sym->m_dst,
1577                                         dst_buf_start,
1578                                         &qat_op_cookie->qat_sgl_list_dst,
1579                                                 qat_req->comn_mid.dst_length);
1580
1581                         if (ret) {
1582                                 PMD_DRV_LOG(ERR, "QAT PMD Cannot "
1583                                                 "fill sgl array");
1584                                 return ret;
1585                         }
1586
1587                         qat_req->comn_mid.src_data_addr =
1588                                 qat_op_cookie->qat_sgl_src_phys_addr;
1589                         qat_req->comn_mid.dest_data_addr =
1590                                         qat_op_cookie->qat_sgl_dst_phys_addr;
1591                 }
1592         } else {
1593                 qat_req->comn_mid.src_data_addr = src_buf_start;
1594                 qat_req->comn_mid.dest_data_addr = dst_buf_start;
1595         }
1596
1597 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
1598         rte_hexdump(stdout, "qat_req:", qat_req,
1599                         sizeof(struct icp_qat_fw_la_bulk_req));
1600         rte_hexdump(stdout, "src_data:",
1601                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
1602                         rte_pktmbuf_data_len(op->sym->m_src));
1603         if (do_cipher) {
1604                 uint8_t *cipher_iv_ptr = rte_crypto_op_ctod_offset(op,
1605                                                 uint8_t *,
1606                                                 ctx->cipher_iv.offset);
1607                 rte_hexdump(stdout, "cipher iv:", cipher_iv_ptr,
1608                                 ctx->cipher_iv.length);
1609         }
1610
1611         if (do_auth) {
1612                 if (ctx->auth_iv.length) {
1613                         uint8_t *auth_iv_ptr = rte_crypto_op_ctod_offset(op,
1614                                                         uint8_t *,
1615                                                         ctx->auth_iv.offset);
1616                         rte_hexdump(stdout, "auth iv:", auth_iv_ptr,
1617                                                 ctx->auth_iv.length);
1618                 }
1619                 rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
1620                                 ctx->digest_length);
1621         }
1622
1623         if (do_aead) {
1624                 rte_hexdump(stdout, "digest:", op->sym->aead.digest.data,
1625                                 ctx->digest_length);
1626                 rte_hexdump(stdout, "aad:", op->sym->aead.aad.data,
1627                                 ctx->aad_len);
1628         }
1629 #endif
1630         return 0;
1631 }
1632
1633 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
1634 {
1635         uint32_t div = data >> shift;
1636         uint32_t mult = div << shift;
1637
1638         return data - mult;
1639 }
1640
1641 int qat_dev_config(__rte_unused struct rte_cryptodev *dev,
1642                 __rte_unused struct rte_cryptodev_config *config)
1643 {
1644         PMD_INIT_FUNC_TRACE();
1645         return 0;
1646 }
1647
1648 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
1649 {
1650         PMD_INIT_FUNC_TRACE();
1651         return 0;
1652 }
1653
1654 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
1655 {
1656         PMD_INIT_FUNC_TRACE();
1657 }
1658
1659 int qat_dev_close(struct rte_cryptodev *dev)
1660 {
1661         int i, ret;
1662
1663         PMD_INIT_FUNC_TRACE();
1664
1665         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1666                 ret = qat_crypto_sym_qp_release(dev, i);
1667                 if (ret < 0)
1668                         return ret;
1669         }
1670
1671         return 0;
1672 }
1673
1674 void qat_dev_info_get(struct rte_cryptodev *dev,
1675                         struct rte_cryptodev_info *info)
1676 {
1677         struct qat_pmd_private *internals = dev->data->dev_private;
1678
1679         PMD_INIT_FUNC_TRACE();
1680         if (info != NULL) {
1681                 info->max_nb_queue_pairs =
1682                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
1683                                 ADF_NUM_BUNDLES_PER_DEV;
1684                 info->feature_flags = dev->feature_flags;
1685                 info->capabilities = internals->qat_dev_capabilities;
1686                 info->sym.max_nb_sessions = internals->max_nb_sessions;
1687                 info->driver_id = cryptodev_qat_driver_id;
1688                 info->pci_dev = RTE_DEV_TO_PCI(dev->device);
1689         }
1690 }
1691
1692 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
1693                 struct rte_cryptodev_stats *stats)
1694 {
1695         int i;
1696         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1697
1698         PMD_INIT_FUNC_TRACE();
1699         if (stats == NULL) {
1700                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
1701                 return;
1702         }
1703         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1704                 if (qp[i] == NULL) {
1705                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
1706                         continue;
1707                 }
1708
1709                 stats->enqueued_count += qp[i]->stats.enqueued_count;
1710                 stats->dequeued_count += qp[i]->stats.dequeued_count;
1711                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
1712                 stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
1713         }
1714 }
1715
1716 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
1717 {
1718         int i;
1719         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1720
1721         PMD_INIT_FUNC_TRACE();
1722         for (i = 0; i < dev->data->nb_queue_pairs; i++)
1723                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
1724         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
1725 }