drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_ether.h>
  50 #include <rte_malloc.h>
  51 #include <rte_launch.h>
  52 #include <rte_eal.h>
  53 #include <rte_per_lcore.h>
  54 #include <rte_lcore.h>
  55 #include <rte_atomic.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_mempool.h>
  58 #include <rte_mbuf.h>
  59 #include <rte_string_fns.h>
  60 #include <rte_spinlock.h>
  61 #include <rte_hexdump.h>
  62 #include <rte_crypto_sym.h>
  63 #include <openssl/evp.h>
  64
  65 #include "qat_logs.h"
  66 #include "qat_algs.h"
  67 #include "qat_crypto.h"
  68 #include "adf_transport_access_macros.h"
  69
  70 #define BYTE_LENGTH    8
  71
  72 static int
  73 qat_is_cipher_alg_supported(enum rte_crypto_cipher_algorithm algo,
  74                 struct qat_pmd_private *internals) {
  75         int i = 0;
  76         const struct rte_cryptodev_capabilities *capability;
  77
  78         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  79                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
  80                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
  81                         continue;
  82
  83                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_CIPHER)
  84                         continue;
  85
  86                 if (capability->sym.cipher.algo == algo)
  87                         return 1;
  88         }
  89         return 0;
  90 }
  91
  92 static int
  93 qat_is_auth_alg_supported(enum rte_crypto_auth_algorithm algo,
  94                 struct qat_pmd_private *internals) {
  95         int i = 0;
  96         const struct rte_cryptodev_capabilities *capability;
  97
  98         while ((capability = &(internals->qat_dev_capabilities[i++]))->op !=
  99                         RTE_CRYPTO_OP_TYPE_UNDEFINED) {
 100                 if (capability->op != RTE_CRYPTO_OP_TYPE_SYMMETRIC)
 101                         continue;
 102
 103                 if (capability->sym.xform_type != RTE_CRYPTO_SYM_XFORM_AUTH)
 104                         continue;
 105
 106                 if (capability->sym.auth.algo == algo)
 107                         return 1;
 108         }
 109         return 0;
 110 }
 111
 112 /** Encrypt a single partial block
 113  *  Depends on openssl libcrypto
 114  *  Uses ECB+XOR to do CFB encryption, same result, more performant
 115  */
 116 static inline int
 117 bpi_cipher_encrypt(uint8_t *src, uint8_t *dst,
 118                 uint8_t *iv, int ivlen, int srclen,
 119                 void *bpi_ctx)
 120 {
 121         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
 122         int encrypted_ivlen;
 123         uint8_t encrypted_iv[16];
 124         int i;
 125
 126         /* ECB method: encrypt the IV, then XOR this with plaintext */
 127         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
 128                                                                 <= 0)
 129                 goto cipher_encrypt_err;
 130
 131         for (i = 0; i < srclen; i++)
 132                 *(dst+i) = *(src+i)^(encrypted_iv[i]);
 133
 134         return 0;
 135
 136 cipher_encrypt_err:
 137         PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt failed");
 138         return -EINVAL;
 139 }
 140
 141 /** Decrypt a single partial block
 142  *  Depends on openssl libcrypto
 143  *  Uses ECB+XOR to do CFB encryption, same result, more performant
 144  */
 145 static inline int
 146 bpi_cipher_decrypt(uint8_t *src, uint8_t *dst,
 147                 uint8_t *iv, int ivlen, int srclen,
 148                 void *bpi_ctx)
 149 {
 150         EVP_CIPHER_CTX *ctx = (EVP_CIPHER_CTX *)bpi_ctx;
 151         int encrypted_ivlen;
 152         uint8_t encrypted_iv[16];
 153         int i;
 154
 155         /* ECB method: encrypt (not decrypt!) the IV, then XOR with plaintext */
 156         if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, iv, ivlen)
 157                                                                 <= 0)
 158                 goto cipher_decrypt_err;
 159
 160         for (i = 0; i < srclen; i++)
 161                 *(dst+i) = *(src+i)^(encrypted_iv[i]);
 162
 163         return 0;
 164
 165 cipher_decrypt_err:
 166         PMD_DRV_LOG(ERR, "libcrypto ECB cipher encrypt for BPI IV failed");
 167         return -EINVAL;
 168 }
 169
 170 /** Creates a context in either AES or DES in ECB mode
 171  *  Depends on openssl libcrypto
 172  */
 173 static void *
 174 bpi_cipher_ctx_init(enum rte_crypto_cipher_algorithm cryptodev_algo,
 175                 enum rte_crypto_cipher_operation direction __rte_unused,
 176                                         uint8_t *key)
 177 {
 178         const EVP_CIPHER *algo = NULL;
 179         EVP_CIPHER_CTX *ctx = EVP_CIPHER_CTX_new();
 180
 181         if (ctx == NULL)
 182                 goto ctx_init_err;
 183
 184         if (cryptodev_algo == RTE_CRYPTO_CIPHER_DES_DOCSISBPI)
 185                 algo = EVP_des_ecb();
 186         else
 187                 algo = EVP_aes_128_ecb();
 188
 189         /* IV will be ECB encrypted whether direction is encrypt or decrypt*/
 190         if (EVP_EncryptInit_ex(ctx, algo, NULL, key, 0) != 1)
 191                 goto ctx_init_err;
 192
 193         return ctx;
 194
 195 ctx_init_err:
 196         if (ctx != NULL)
 197                 EVP_CIPHER_CTX_free(ctx);
 198         return NULL;
 199 }
 200
 201 /** Frees a context previously created
 202  *  Depends on openssl libcrypto
 203  */
 204 static void
 205 bpi_cipher_ctx_free(void *bpi_ctx)
 206 {
 207         if (bpi_ctx != NULL)
 208                 EVP_CIPHER_CTX_free((EVP_CIPHER_CTX *)bpi_ctx);
 209 }
 210
 211 static inline uint32_t
 212 adf_modulo(uint32_t data, uint32_t shift);
 213
 214 static inline int
 215 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
 216                 struct qat_crypto_op_cookie *qat_op_cookie);
 217
 218 void qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 219                 void *session)
 220 {
 221         struct qat_session *sess = session;
 222         phys_addr_t cd_paddr;
 223
 224         PMD_INIT_FUNC_TRACE();
 225         if (sess) {
 226                 if (sess->bpi_ctx) {
 227                         bpi_cipher_ctx_free(sess->bpi_ctx);
 228                         sess->bpi_ctx = NULL;
 229                 }
 230                 cd_paddr = sess->cd_paddr;
 231                 memset(sess, 0, qat_crypto_sym_get_session_private_size(dev));
 232                 sess->cd_paddr = cd_paddr;
 233         } else
 234                 PMD_DRV_LOG(ERR, "NULL session");
 235 }
 236
 237 static int
 238 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 239 {
 240         /* Cipher Only */
 241         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 242                 return ICP_QAT_FW_LA_CMD_CIPHER;
 243
 244         /* Authentication Only */
 245         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 246                 return ICP_QAT_FW_LA_CMD_AUTH;
 247
 248         if (xform->next == NULL)
 249                 return -1;
 250
 251         /* Cipher then Authenticate */
 252         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 253                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 254                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 255
 256         /* Authenticate then Cipher */
 257         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 258                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 259                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 260
 261         return -1;
 262 }
 263
 264 static struct rte_crypto_auth_xform *
 265 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 266 {
 267         do {
 268                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 269                         return &xform->auth;
 270
 271                 xform = xform->next;
 272         } while (xform);
 273
 274         return NULL;
 275 }
 276
 277 static struct rte_crypto_cipher_xform *
 278 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 279 {
 280         do {
 281                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 282                         return &xform->cipher;
 283
 284                 xform = xform->next;
 285         } while (xform);
 286
 287         return NULL;
 288 }
 289 void *
 290 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 291                 struct rte_crypto_sym_xform *xform, void *session_private)
 292 {
 293         struct qat_session *session = session_private;
 294         struct qat_pmd_private *internals = dev->data->dev_private;
 295         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 296
 297         /* Get cipher xform from crypto xform chain */
 298         cipher_xform = qat_get_cipher_xform(xform);
 299
 300         switch (cipher_xform->algo) {
 301         case RTE_CRYPTO_CIPHER_AES_CBC:
 302                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 303                                 &session->qat_cipher_alg) != 0) {
 304                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 305                         goto error_out;
 306                 }
 307                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 308                 break;
 309         case RTE_CRYPTO_CIPHER_AES_GCM:
 310                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 311                                 &session->qat_cipher_alg) != 0) {
 312                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 313                         goto error_out;
 314                 }
 315                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 316                 break;
 317         case RTE_CRYPTO_CIPHER_AES_CTR:
 318                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 319                                 &session->qat_cipher_alg) != 0) {
 320                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 321                         goto error_out;
 322                 }
 323                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 324                 break;
 325         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 326                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 327                                         &session->qat_cipher_alg) != 0) {
 328                         PMD_DRV_LOG(ERR, "Invalid SNOW 3G cipher key size");
 329                         goto error_out;
 330                 }
 331                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 332                 break;
 333         case RTE_CRYPTO_CIPHER_NULL:
 334                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 335                 break;
 336         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 337                 if (qat_alg_validate_kasumi_key(cipher_xform->key.length,
 338                                         &session->qat_cipher_alg) != 0) {
 339                         PMD_DRV_LOG(ERR, "Invalid KASUMI cipher key size");
 340                         goto error_out;
 341                 }
 342                 session->qat_mode = ICP_QAT_HW_CIPHER_F8_MODE;
 343                 break;
 344         case RTE_CRYPTO_CIPHER_3DES_CBC:
 345                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 346                                 &session->qat_cipher_alg) != 0) {
 347                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 348                         goto error_out;
 349                 }
 350                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 351                 break;
 352         case RTE_CRYPTO_CIPHER_DES_CBC:
 353                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 354                                 &session->qat_cipher_alg) != 0) {
 355                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 356                         goto error_out;
 357                 }
 358                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 359                 break;
 360         case RTE_CRYPTO_CIPHER_3DES_CTR:
 361                 if (qat_alg_validate_3des_key(cipher_xform->key.length,
 362                                 &session->qat_cipher_alg) != 0) {
 363                         PMD_DRV_LOG(ERR, "Invalid 3DES cipher key size");
 364                         goto error_out;
 365                 }
 366                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 367                 break;
 368         case RTE_CRYPTO_CIPHER_DES_DOCSISBPI:
 369                 session->bpi_ctx = bpi_cipher_ctx_init(
 370                                         cipher_xform->algo,
 371                                         cipher_xform->op,
 372                                         cipher_xform->key.data);
 373                 if (session->bpi_ctx == NULL) {
 374                         PMD_DRV_LOG(ERR, "failed to create DES BPI ctx");
 375                         goto error_out;
 376                 }
 377                 if (qat_alg_validate_des_key(cipher_xform->key.length,
 378                                 &session->qat_cipher_alg) != 0) {
 379                         PMD_DRV_LOG(ERR, "Invalid DES cipher key size");
 380                         goto error_out;
 381                 }
 382                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 383                 break;
 384         case RTE_CRYPTO_CIPHER_AES_DOCSISBPI:
 385                 session->bpi_ctx = bpi_cipher_ctx_init(
 386                                         cipher_xform->algo,
 387                                         cipher_xform->op,
 388                                         cipher_xform->key.data);
 389                 if (session->bpi_ctx == NULL) {
 390                         PMD_DRV_LOG(ERR, "failed to create AES BPI ctx");
 391                         goto error_out;
 392                 }
 393                 if (qat_alg_validate_aes_docsisbpi_key(cipher_xform->key.length,
 394                                 &session->qat_cipher_alg) != 0) {
 395                         PMD_DRV_LOG(ERR, "Invalid AES DOCSISBPI key size");
 396                         goto error_out;
 397                 }
 398                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 399                 break;
 400         case RTE_CRYPTO_CIPHER_ZUC_EEA3:
 401                 if (!qat_is_cipher_alg_supported(
 402                         cipher_xform->algo, internals)) {
 403                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 404                                 rte_crypto_cipher_algorithm_strings
 405                                         [cipher_xform->algo]);
 406                         goto error_out;
 407                 }
 408                 if (qat_alg_validate_zuc_key(cipher_xform->key.length,
 409                                 &session->qat_cipher_alg) != 0) {
 410                         PMD_DRV_LOG(ERR, "Invalid ZUC cipher key size");
 411                         goto error_out;
 412                 }
 413                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 414                 break;
 415         case RTE_CRYPTO_CIPHER_3DES_ECB:
 416         case RTE_CRYPTO_CIPHER_AES_ECB:
 417         case RTE_CRYPTO_CIPHER_AES_CCM:
 418         case RTE_CRYPTO_CIPHER_AES_F8:
 419         case RTE_CRYPTO_CIPHER_AES_XTS:
 420         case RTE_CRYPTO_CIPHER_ARC4:
 421                 PMD_DRV_LOG(ERR, "Crypto QAT PMD: Unsupported Cipher alg %u",
 422                                 cipher_xform->algo);
 423                 goto error_out;
 424         default:
 425                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 426                                 cipher_xform->algo);
 427                 goto error_out;
 428         }
 429
 430         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 431                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 432         else
 433                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 434
 435         if (qat_alg_aead_session_create_content_desc_cipher(session,
 436                                                 cipher_xform->key.data,
 437                                                 cipher_xform->key.length))
 438                 goto error_out;
 439
 440         return session;
 441
 442 error_out:
 443         if (session->bpi_ctx) {
 444                 bpi_cipher_ctx_free(session->bpi_ctx);
 445                 session->bpi_ctx = NULL;
 446         }
 447         return NULL;
 448 }
 449
 450
 451 void *
 452 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 453                 struct rte_crypto_sym_xform *xform, void *session_private)
 454 {
 455         struct qat_session *session = session_private;
 456
 457         int qat_cmd_id;
 458         PMD_INIT_FUNC_TRACE();
 459
 460         /* Get requested QAT command id */
 461         qat_cmd_id = qat_get_cmd_id(xform);
 462         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 463                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 464                 goto error_out;
 465         }
 466         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 467         switch (session->qat_cmd) {
 468         case ICP_QAT_FW_LA_CMD_CIPHER:
 469         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 470                 break;
 471         case ICP_QAT_FW_LA_CMD_AUTH:
 472         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 473                 break;
 474         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 475         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 476         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 477                 break;
 478         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 479         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 480         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 481                 break;
 482         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 483         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 484         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 485         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 486         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 487         case ICP_QAT_FW_LA_CMD_MGF1:
 488         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 489         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 490         case ICP_QAT_FW_LA_CMD_DELIMITER:
 491         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 492                 session->qat_cmd);
 493                 goto error_out;
 494         default:
 495         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 496                 session->qat_cmd);
 497                 goto error_out;
 498         }
 499
 500         return session;
 501
 502 error_out:
 503         return NULL;
 504 }
 505
 506 struct qat_session *
 507 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 508                                 struct rte_crypto_sym_xform *xform,
 509                                 struct qat_session *session_private)
 510 {
 511
 512         struct qat_session *session = session_private;
 513         struct rte_crypto_auth_xform *auth_xform = NULL;
 514         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 515         struct qat_pmd_private *internals = dev->data->dev_private;
 516         auth_xform = qat_get_auth_xform(xform);
 517
 518         switch (auth_xform->algo) {
 519         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 520                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 521                 break;
 522         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 523                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA224;
 524                 break;
 525         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 526                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 527                 break;
 528         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 529                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA384;
 530                 break;
 531         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 532                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 533                 break;
 534         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 535                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 536                 break;
 537         case RTE_CRYPTO_AUTH_AES_GCM:
 538                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 539                 break;
 540         case RTE_CRYPTO_AUTH_AES_GMAC:
 541                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 542                 break;
 543         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 544                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 545                 break;
 546         case RTE_CRYPTO_AUTH_MD5_HMAC:
 547                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_MD5;
 548                 break;
 549         case RTE_CRYPTO_AUTH_NULL:
 550                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_NULL;
 551                 break;
 552         case RTE_CRYPTO_AUTH_KASUMI_F9:
 553                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_KASUMI_F9;
 554                 break;
 555         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 556                 if (!qat_is_auth_alg_supported(auth_xform->algo, internals)) {
 557                         PMD_DRV_LOG(ERR, "%s not supported on this device",
 558                                 rte_crypto_auth_algorithm_strings
 559                                 [auth_xform->algo]);
 560                         goto error_out;
 561                 }
 562                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3;
 563                 break;
 564         case RTE_CRYPTO_AUTH_SHA1:
 565         case RTE_CRYPTO_AUTH_SHA256:
 566         case RTE_CRYPTO_AUTH_SHA512:
 567         case RTE_CRYPTO_AUTH_SHA224:
 568         case RTE_CRYPTO_AUTH_SHA384:
 569         case RTE_CRYPTO_AUTH_MD5:
 570         case RTE_CRYPTO_AUTH_AES_CCM:
 571         case RTE_CRYPTO_AUTH_AES_CMAC:
 572         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 573                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 574                                 auth_xform->algo);
 575                 goto error_out;
 576         default:
 577                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 578                                 auth_xform->algo);
 579                 goto error_out;
 580         }
 581         cipher_xform = qat_get_cipher_xform(xform);
 582
 583         if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
 584                         (session->qat_hash_alg ==
 585                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_64))  {
 586                 if (qat_alg_aead_session_create_content_desc_auth(session,
 587                                 cipher_xform->key.data,
 588                                 cipher_xform->key.length,
 589                                 auth_xform->add_auth_data_length,
 590                                 auth_xform->digest_length,
 591                                 auth_xform->op))
 592                         goto error_out;
 593         } else {
 594                 if (qat_alg_aead_session_create_content_desc_auth(session,
 595                                 auth_xform->key.data,
 596                                 auth_xform->key.length,
 597                                 auth_xform->add_auth_data_length,
 598                                 auth_xform->digest_length,
 599                                 auth_xform->op))
 600                         goto error_out;
 601         }
 602         return session;
 603
 604 error_out:
 605         return NULL;
 606 }
 607
 608 unsigned qat_crypto_sym_get_session_private_size(
 609                 struct rte_cryptodev *dev __rte_unused)
 610 {
 611         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 612 }
 613
 614 static inline uint32_t
 615 qat_bpicipher_preprocess(struct qat_session *ctx,
 616                                 struct rte_crypto_op *op)
 617 {
 618         uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
 619         struct rte_crypto_sym_op *sym_op = op->sym;
 620         uint8_t last_block_len = sym_op->cipher.data.length % block_len;
 621
 622         if (last_block_len &&
 623                         ctx->qat_dir == ICP_QAT_HW_CIPHER_DECRYPT) {
 624
 625                 /* Decrypt last block */
 626                 uint8_t *last_block, *dst, *iv;
 627                 uint32_t last_block_offset = sym_op->cipher.data.offset +
 628                                 sym_op->cipher.data.length - last_block_len;
 629                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
 630                                 uint8_t *, last_block_offset);
 631
 632                 if (unlikely(sym_op->m_dst != NULL))
 633                         /* out-of-place operation (OOP) */
 634                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
 635                                                 uint8_t *, last_block_offset);
 636                 else
 637                         dst = last_block;
 638
 639                 if (last_block_len < sym_op->cipher.data.length)
 640                         /* use previous block ciphertext as IV */
 641                         iv = last_block - block_len;
 642                 else
 643                         /* runt block, i.e. less than one full block */
 644                         iv = sym_op->cipher.iv.data;
 645
 646 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 647                 rte_hexdump(stdout, "BPI: src before pre-process:", last_block,
 648                         last_block_len);
 649                 if (sym_op->m_dst != NULL)
 650                         rte_hexdump(stdout, "BPI: dst before pre-process:", dst,
 651                                 last_block_len);
 652 #endif
 653                 bpi_cipher_decrypt(last_block, dst, iv, block_len,
 654                                 last_block_len, ctx->bpi_ctx);
 655 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 656                 rte_hexdump(stdout, "BPI: src after pre-process:", last_block,
 657                         last_block_len);
 658                 if (sym_op->m_dst != NULL)
 659                         rte_hexdump(stdout, "BPI: dst after pre-process:", dst,
 660                                 last_block_len);
 661 #endif
 662         }
 663
 664         return sym_op->cipher.data.length - last_block_len;
 665 }
 666
 667 static inline uint32_t
 668 qat_bpicipher_postprocess(struct qat_session *ctx,
 669                                 struct rte_crypto_op *op)
 670 {
 671         uint8_t block_len = qat_cipher_get_block_size(ctx->qat_cipher_alg);
 672         struct rte_crypto_sym_op *sym_op = op->sym;
 673         uint8_t last_block_len = sym_op->cipher.data.length % block_len;
 674
 675         if (last_block_len > 0 &&
 676                         ctx->qat_dir == ICP_QAT_HW_CIPHER_ENCRYPT) {
 677
 678                 /* Encrypt last block */
 679                 uint8_t *last_block, *dst, *iv;
 680                 uint32_t last_block_offset;
 681
 682                 last_block_offset = sym_op->cipher.data.offset +
 683                                 sym_op->cipher.data.length - last_block_len;
 684                 last_block = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_src,
 685                                 uint8_t *, last_block_offset);
 686
 687                 if (unlikely(sym_op->m_dst != NULL))
 688                         /* out-of-place operation (OOP) */
 689                         dst = (uint8_t *) rte_pktmbuf_mtod_offset(sym_op->m_dst,
 690                                                 uint8_t *, last_block_offset);
 691                 else
 692                         dst = last_block;
 693
 694                 if (last_block_len < sym_op->cipher.data.length)
 695                         /* use previous block ciphertext as IV */
 696                         iv = dst - block_len;
 697                 else
 698                         /* runt block, i.e. less than one full block */
 699                         iv = sym_op->cipher.iv.data;
 700
 701 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 702                 rte_hexdump(stdout, "BPI: src before post-process:", last_block,
 703                         last_block_len);
 704                 if (sym_op->m_dst != NULL)
 705                         rte_hexdump(stdout, "BPI: dst before post-process:",
 706                                         dst, last_block_len);
 707 #endif
 708                 bpi_cipher_encrypt(last_block, dst, iv, block_len,
 709                                 last_block_len, ctx->bpi_ctx);
 710 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 711                 rte_hexdump(stdout, "BPI: src after post-process:", last_block,
 712                         last_block_len);
 713                 if (sym_op->m_dst != NULL)
 714                         rte_hexdump(stdout, "BPI: dst after post-process:", dst,
 715                                 last_block_len);
 716 #endif
 717         }
 718         return sym_op->cipher.data.length - last_block_len;
 719 }
 720
 721 uint16_t
 722 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 723                 uint16_t nb_ops)
 724 {
 725         register struct qat_queue *queue;
 726         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 727         register uint32_t nb_ops_sent = 0;
 728         register struct rte_crypto_op **cur_op = ops;
 729         register int ret;
 730         uint16_t nb_ops_possible = nb_ops;
 731         register uint8_t *base_addr;
 732         register uint32_t tail;
 733         int overflow;
 734
 735         if (unlikely(nb_ops == 0))
 736                 return 0;
 737
 738         /* read params used a lot in main loop into registers */
 739         queue = &(tmp_qp->tx_q);
 740         base_addr = (uint8_t *)queue->base_addr;
 741         tail = queue->tail;
 742
 743         /* Find how many can actually fit on the ring */
 744         overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops)
 745                                 - queue->max_inflights;
 746         if (overflow > 0) {
 747                 rte_atomic16_sub(&tmp_qp->inflights16, overflow);
 748                 nb_ops_possible = nb_ops - overflow;
 749                 if (nb_ops_possible == 0)
 750                         return 0;
 751         }
 752
 753         while (nb_ops_sent != nb_ops_possible) {
 754                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail,
 755                                 tmp_qp->op_cookies[tail / queue->msg_size]);
 756                 if (ret != 0) {
 757                         tmp_qp->stats.enqueue_err_count++;
 758                         /*
 759                          * This message cannot be enqueued,
 760                          * decrease number of ops that wasnt sent
 761                          */
 762                         rte_atomic16_sub(&tmp_qp->inflights16,
 763                                         nb_ops_possible - nb_ops_sent);
 764                         if (nb_ops_sent == 0)
 765                                 return 0;
 766                         goto kick_tail;
 767                 }
 768
 769                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 770                 nb_ops_sent++;
 771                 cur_op++;
 772         }
 773 kick_tail:
 774         WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number,
 775                         queue->hw_queue_number, tail);
 776         queue->tail = tail;
 777         tmp_qp->stats.enqueued_count += nb_ops_sent;
 778         return nb_ops_sent;
 779 }
 780
 781 uint16_t
 782 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 783                 uint16_t nb_ops)
 784 {
 785         struct qat_queue *queue;
 786         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 787         uint32_t msg_counter = 0;
 788         struct rte_crypto_op *rx_op;
 789         struct icp_qat_fw_comn_resp *resp_msg;
 790
 791         queue = &(tmp_qp->rx_q);
 792         resp_msg = (struct icp_qat_fw_comn_resp *)
 793                         ((uint8_t *)queue->base_addr + queue->head);
 794
 795         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 796                         msg_counter != nb_ops) {
 797                 rx_op = (struct rte_crypto_op *)(uintptr_t)
 798                                 (resp_msg->opaque_data);
 799
 800 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 801                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
 802                         sizeof(struct icp_qat_fw_comn_resp));
 803
 804 #endif
 805                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
 806                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
 807                                         resp_msg->comn_hdr.comn_status)) {
 808                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 809                 } else {
 810                         struct qat_session *sess = (struct qat_session *)
 811                                                 (rx_op->sym->session->_private);
 812                         if (sess->bpi_ctx)
 813                                 qat_bpicipher_postprocess(sess, rx_op);
 814                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 815                 }
 816
 817                 *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG;
 818                 queue->head = adf_modulo(queue->head +
 819                                 queue->msg_size,
 820                                 ADF_RING_SIZE_MODULO(queue->queue_size));
 821                 resp_msg = (struct icp_qat_fw_comn_resp *)
 822                                         ((uint8_t *)queue->base_addr +
 823                                                         queue->head);
 824                 *ops = rx_op;
 825                 ops++;
 826                 msg_counter++;
 827         }
 828         if (msg_counter > 0) {
 829                 WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr,
 830                                         queue->hw_bundle_number,
 831                                         queue->hw_queue_number, queue->head);
 832                 rte_atomic16_sub(&tmp_qp->inflights16, msg_counter);
 833                 tmp_qp->stats.dequeued_count += msg_counter;
 834         }
 835         return msg_counter;
 836 }
 837
 838 static inline int
 839 qat_sgl_fill_array(struct rte_mbuf *buf, uint64_t buff_start,
 840                 struct qat_alg_buf_list *list, uint32_t data_len)
 841 {
 842         int nr = 1;
 843
 844         uint32_t buf_len = rte_pktmbuf_mtophys(buf) -
 845                         buff_start + rte_pktmbuf_data_len(buf);
 846
 847         list->bufers[0].addr = buff_start;
 848         list->bufers[0].resrvd = 0;
 849         list->bufers[0].len = buf_len;
 850
 851         if (data_len <= buf_len) {
 852                 list->num_bufs = nr;
 853                 list->bufers[0].len = data_len;
 854                 return 0;
 855         }
 856
 857         buf = buf->next;
 858         while (buf) {
 859                 if (unlikely(nr == QAT_SGL_MAX_NUMBER)) {
 860                         PMD_DRV_LOG(ERR, "QAT PMD exceeded size of QAT SGL"
 861                                         " entry(%u)",
 862                                         QAT_SGL_MAX_NUMBER);
 863                         return -EINVAL;
 864                 }
 865
 866                 list->bufers[nr].len = rte_pktmbuf_data_len(buf);
 867                 list->bufers[nr].resrvd = 0;
 868                 list->bufers[nr].addr = rte_pktmbuf_mtophys(buf);
 869
 870                 buf_len += list->bufers[nr].len;
 871                 buf = buf->next;
 872
 873                 if (buf_len > data_len) {
 874                         list->bufers[nr].len -=
 875                                 buf_len - data_len;
 876                         buf = NULL;
 877                 }
 878                 ++nr;
 879         }
 880         list->num_bufs = nr;
 881
 882         return 0;
 883 }
 884
 885 static inline int
 886 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg,
 887                 struct qat_crypto_op_cookie *qat_op_cookie)
 888 {
 889         int ret = 0;
 890         struct qat_session *ctx;
 891         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 892         struct icp_qat_fw_la_auth_req_params *auth_param;
 893         register struct icp_qat_fw_la_bulk_req *qat_req;
 894         uint8_t do_auth = 0, do_cipher = 0;
 895         uint32_t cipher_len = 0, cipher_ofs = 0;
 896         uint32_t auth_len = 0, auth_ofs = 0;
 897         uint32_t min_ofs = 0;
 898         uint64_t src_buf_start = 0, dst_buf_start = 0;
 899         uint8_t do_sgl = 0;
 900
 901
 902 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 903         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 904                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
 905                                 "operation requests, op (%p) is not a "
 906                                 "symmetric operation.", op);
 907                 return -EINVAL;
 908         }
 909 #endif
 910         if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) {
 911                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
 912                                 " requests, op (%p) is sessionless.", op);
 913                 return -EINVAL;
 914         }
 915
 916         if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) {
 917                 PMD_DRV_LOG(ERR, "Session was not created for this device");
 918                 return -EINVAL;
 919         }
 920
 921         ctx = (struct qat_session *)op->sym->session->_private;
 922         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 923         rte_mov128((uint8_t *)qat_req, (const uint8_t *)&(ctx->fw_req));
 924         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
 925         cipher_param = (void *)&qat_req->serv_specif_rqpars;
 926         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 927
 928         if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER ||
 929                 ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER_HASH) {
 930                 do_auth = 1;
 931                 do_cipher = 1;
 932         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_AUTH) {
 933                 do_auth = 1;
 934                 do_cipher = 0;
 935         } else if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_CIPHER) {
 936                 do_auth = 0;
 937                 do_cipher = 1;
 938         }
 939
 940         if (do_cipher) {
 941
 942                 if (ctx->qat_cipher_alg ==
 943                                          ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2 ||
 944                         ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_KASUMI ||
 945                         ctx->qat_cipher_alg ==
 946                                 ICP_QAT_HW_CIPHER_ALGO_ZUC_3G_128_EEA3) {
 947
 948                         if (unlikely(
 949                                 (cipher_param->cipher_length % BYTE_LENGTH != 0)
 950                                  || (cipher_param->cipher_offset
 951                                                         % BYTE_LENGTH != 0))) {
 952                                 PMD_DRV_LOG(ERR,
 953                   "SNOW3G/KASUMI/ZUC in QAT PMD only supports byte aligned values");
 954                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 955                                 return -EINVAL;
 956                         }
 957                         cipher_len = op->sym->cipher.data.length >> 3;
 958                         cipher_ofs = op->sym->cipher.data.offset >> 3;
 959
 960                 } else if (ctx->bpi_ctx) {
 961                         /* DOCSIS - only send complete blocks to device
 962                          * Process any partial block using CFB mode.
 963                          * Even if 0 complete blocks, still send this to device
 964                          * to get into rx queue for post-process and dequeuing
 965                          */
 966                         cipher_len = qat_bpicipher_preprocess(ctx, op);
 967                         cipher_ofs = op->sym->cipher.data.offset;
 968                 } else {
 969                         cipher_len = op->sym->cipher.data.length;
 970                         cipher_ofs = op->sym->cipher.data.offset;
 971                 }
 972
 973                 /* copy IV into request if it fits */
 974                 /*
 975                  * If IV length is zero do not copy anything but still
 976                  * use request descriptor embedded IV
 977                  *
 978                  */
 979                 if (op->sym->cipher.iv.length) {
 980                         if (op->sym->cipher.iv.length <=
 981                                         sizeof(cipher_param->u.cipher_IV_array)) {
 982                                 rte_memcpy(cipher_param->u.cipher_IV_array,
 983                                                 op->sym->cipher.iv.data,
 984                                                 op->sym->cipher.iv.length);
 985                         } else {
 986                                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
 987                                                 qat_req->comn_hdr.serv_specif_flags,
 988                                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
 989                                 cipher_param->u.s.cipher_IV_ptr =
 990                                                 op->sym->cipher.iv.phys_addr;
 991                         }
 992                 }
 993                 min_ofs = cipher_ofs;
 994         }
 995
 996         if (do_auth) {
 997
 998                 if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2 ||
 999                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_KASUMI_F9 ||
1000                         ctx->qat_hash_alg ==
1001                                 ICP_QAT_HW_AUTH_ALGO_ZUC_3G_128_EIA3) {
1002                         if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0)
1003                                 || (auth_param->auth_len % BYTE_LENGTH != 0))) {
1004                                 PMD_DRV_LOG(ERR,
1005                 "For SNOW3G/KASUMI/ZUC, QAT PMD only supports byte aligned values");
1006                                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
1007                                 return -EINVAL;
1008                         }
1009                         auth_ofs = op->sym->auth.data.offset >> 3;
1010                         auth_len = op->sym->auth.data.length >> 3;
1011
1012                         if (ctx->qat_hash_alg ==
1013                                         ICP_QAT_HW_AUTH_ALGO_KASUMI_F9) {
1014                                 if (do_cipher) {
1015                                         auth_len = auth_len + auth_ofs + 1 -
1016                                                 ICP_QAT_HW_KASUMI_BLK_SZ;
1017                                         auth_ofs = ICP_QAT_HW_KASUMI_BLK_SZ;
1018                                 } else {
1019                                         auth_len = auth_len + auth_ofs + 1;
1020                                         auth_ofs = 0;
1021                                 }
1022                         }
1023
1024                 } else if (ctx->qat_hash_alg ==
1025                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1026                                 ctx->qat_hash_alg ==
1027                                         ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
1028                         auth_ofs = op->sym->cipher.data.offset;
1029                         auth_len = op->sym->cipher.data.length;
1030                 } else {
1031                         auth_ofs = op->sym->auth.data.offset;
1032                         auth_len = op->sym->auth.data.length;
1033                 }
1034                 min_ofs = auth_ofs;
1035
1036                 auth_param->auth_res_addr = op->sym->auth.digest.phys_addr;
1037
1038                 auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr;
1039
1040         }
1041
1042         if (op->sym->m_src->next || (op->sym->m_dst && op->sym->m_dst->next))
1043                 do_sgl = 1;
1044
1045         /* adjust for chain case */
1046         if (do_cipher && do_auth)
1047                 min_ofs = cipher_ofs < auth_ofs ? cipher_ofs : auth_ofs;
1048
1049         if (unlikely(min_ofs >= rte_pktmbuf_data_len(op->sym->m_src) && do_sgl))
1050                 min_ofs = 0;
1051
1052         if (unlikely(op->sym->m_dst != NULL)) {
1053                 /* Out-of-place operation (OOP)
1054                  * Don't align DMA start. DMA the minimum data-set
1055                  * so as not to overwrite data in dest buffer
1056                  */
1057                 src_buf_start =
1058                         rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs);
1059                 dst_buf_start =
1060                         rte_pktmbuf_mtophys_offset(op->sym->m_dst, min_ofs);
1061
1062         } else {
1063                 /* In-place operation
1064                  * Start DMA at nearest aligned address below min_ofs
1065                  */
1066                 src_buf_start =
1067                         rte_pktmbuf_mtophys_offset(op->sym->m_src, min_ofs)
1068                                                 & QAT_64_BTYE_ALIGN_MASK;
1069
1070                 if (unlikely((rte_pktmbuf_mtophys(op->sym->m_src) -
1071                                         rte_pktmbuf_headroom(op->sym->m_src))
1072                                                         > src_buf_start)) {
1073                         /* alignment has pushed addr ahead of start of mbuf
1074                          * so revert and take the performance hit
1075                          */
1076                         src_buf_start =
1077                                 rte_pktmbuf_mtophys_offset(op->sym->m_src,
1078                                                                 min_ofs);
1079                 }
1080                 dst_buf_start = src_buf_start;
1081         }
1082
1083         if (do_cipher) {
1084                 cipher_param->cipher_offset =
1085                                 (uint32_t)rte_pktmbuf_mtophys_offset(
1086                                 op->sym->m_src, cipher_ofs) - src_buf_start;
1087                 cipher_param->cipher_length = cipher_len;
1088         } else {
1089                 cipher_param->cipher_offset = 0;
1090                 cipher_param->cipher_length = 0;
1091         }
1092         if (do_auth) {
1093                 auth_param->auth_off = (uint32_t)rte_pktmbuf_mtophys_offset(
1094                                 op->sym->m_src, auth_ofs) - src_buf_start;
1095                 auth_param->auth_len = auth_len;
1096         } else {
1097                 auth_param->auth_off = 0;
1098                 auth_param->auth_len = 0;
1099         }
1100         qat_req->comn_mid.dst_length =
1101                 qat_req->comn_mid.src_length =
1102                 (cipher_param->cipher_offset + cipher_param->cipher_length)
1103                 > (auth_param->auth_off + auth_param->auth_len) ?
1104                 (cipher_param->cipher_offset + cipher_param->cipher_length)
1105                 : (auth_param->auth_off + auth_param->auth_len);
1106
1107         if (do_sgl) {
1108
1109                 ICP_QAT_FW_COMN_PTR_TYPE_SET(qat_req->comn_hdr.comn_req_flags,
1110                                 QAT_COMN_PTR_TYPE_SGL);
1111                 ret = qat_sgl_fill_array(op->sym->m_src, src_buf_start,
1112                                 &qat_op_cookie->qat_sgl_list_src,
1113                                 qat_req->comn_mid.src_length);
1114                 if (ret) {
1115                         PMD_DRV_LOG(ERR, "QAT PMD Cannot fill sgl array");
1116                         return ret;
1117                 }
1118
1119                 if (likely(op->sym->m_dst == NULL))
1120                         qat_req->comn_mid.dest_data_addr =
1121                                 qat_req->comn_mid.src_data_addr =
1122                                 qat_op_cookie->qat_sgl_src_phys_addr;
1123                 else {
1124                         ret = qat_sgl_fill_array(op->sym->m_dst,
1125                                         dst_buf_start,
1126                                         &qat_op_cookie->qat_sgl_list_dst,
1127                                                 qat_req->comn_mid.dst_length);
1128
1129                         if (ret) {
1130                                 PMD_DRV_LOG(ERR, "QAT PMD Cannot "
1131                                                 "fill sgl array");
1132                                 return ret;
1133                         }
1134
1135                         qat_req->comn_mid.src_data_addr =
1136                                 qat_op_cookie->qat_sgl_src_phys_addr;
1137                         qat_req->comn_mid.dest_data_addr =
1138                                         qat_op_cookie->qat_sgl_dst_phys_addr;
1139                 }
1140         } else {
1141                 qat_req->comn_mid.src_data_addr = src_buf_start;
1142                 qat_req->comn_mid.dest_data_addr = dst_buf_start;
1143         }
1144
1145         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
1146                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
1147                 if (op->sym->cipher.iv.length == 12) {
1148                         /*
1149                          * For GCM a 12 bit IV is allowed,
1150                          * but we need to inform the f/w
1151                          */
1152                         ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
1153                                 qat_req->comn_hdr.serv_specif_flags,
1154                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
1155                 }
1156                 if (op->sym->cipher.data.length == 0) {
1157                         /*
1158                          * GMAC
1159                          */
1160                         qat_req->comn_mid.dest_data_addr =
1161                                 qat_req->comn_mid.src_data_addr =
1162                                                 op->sym->auth.aad.phys_addr;
1163                         qat_req->comn_mid.dst_length =
1164                                 qat_req->comn_mid.src_length =
1165                                         rte_pktmbuf_data_len(op->sym->m_src);
1166                         cipher_param->cipher_length = 0;
1167                         cipher_param->cipher_offset = 0;
1168                         auth_param->u1.aad_adr = 0;
1169                         auth_param->auth_len = op->sym->auth.aad.length;
1170                         auth_param->auth_off = op->sym->auth.data.offset;
1171                         auth_param->u2.aad_sz = 0;
1172                 }
1173         }
1174
1175 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
1176         rte_hexdump(stdout, "qat_req:", qat_req,
1177                         sizeof(struct icp_qat_fw_la_bulk_req));
1178         rte_hexdump(stdout, "src_data:",
1179                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
1180                         rte_pktmbuf_data_len(op->sym->m_src));
1181         rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data,
1182                         op->sym->cipher.iv.length);
1183         rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
1184                         op->sym->auth.digest.length);
1185         rte_hexdump(stdout, "aad:", op->sym->auth.aad.data,
1186                         op->sym->auth.aad.length);
1187 #endif
1188         return 0;
1189 }
1190
1191 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
1192 {
1193         uint32_t div = data >> shift;
1194         uint32_t mult = div << shift;
1195
1196         return data - mult;
1197 }
1198
1199 void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess)
1200 {
1201         struct rte_cryptodev_sym_session *sess = sym_sess;
1202         struct qat_session *s = (void *)sess->_private;
1203
1204         PMD_INIT_FUNC_TRACE();
1205         s->cd_paddr = rte_mempool_virt2phy(mp, sess) +
1206                 offsetof(struct qat_session, cd) +
1207                 offsetof(struct rte_cryptodev_sym_session, _private);
1208 }
1209
1210 int qat_dev_config(__rte_unused struct rte_cryptodev *dev,
1211                 __rte_unused struct rte_cryptodev_config *config)
1212 {
1213         PMD_INIT_FUNC_TRACE();
1214         return 0;
1215 }
1216
1217 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
1218 {
1219         PMD_INIT_FUNC_TRACE();
1220         return 0;
1221 }
1222
1223 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
1224 {
1225         PMD_INIT_FUNC_TRACE();
1226 }
1227
1228 int qat_dev_close(struct rte_cryptodev *dev)
1229 {
1230         int i, ret;
1231
1232         PMD_INIT_FUNC_TRACE();
1233
1234         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1235                 ret = qat_crypto_sym_qp_release(dev, i);
1236                 if (ret < 0)
1237                         return ret;
1238         }
1239
1240         return 0;
1241 }
1242
1243 void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev,
1244                                 struct rte_cryptodev_info *info)
1245 {
1246         struct qat_pmd_private *internals = dev->data->dev_private;
1247
1248         PMD_INIT_FUNC_TRACE();
1249         if (info != NULL) {
1250                 info->max_nb_queue_pairs =
1251                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
1252                                 ADF_NUM_BUNDLES_PER_DEV;
1253                 info->feature_flags = dev->feature_flags;
1254                 info->capabilities = internals->qat_dev_capabilities;
1255                 info->sym.max_nb_sessions = internals->max_nb_sessions;
1256                 info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD;
1257         }
1258 }
1259
1260 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
1261                 struct rte_cryptodev_stats *stats)
1262 {
1263         int i;
1264         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1265
1266         PMD_INIT_FUNC_TRACE();
1267         if (stats == NULL) {
1268                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
1269                 return;
1270         }
1271         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
1272                 if (qp[i] == NULL) {
1273                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
1274                         continue;
1275                 }
1276
1277                 stats->enqueued_count += qp[i]->stats.enqueued_count;
1278                 stats->dequeued_count += qp[i]->stats.dequeued_count;
1279                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
1280                 stats->dequeue_err_count += qp[i]->stats.dequeue_err_count;
1281         }
1282 }
1283
1284 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
1285 {
1286         int i;
1287         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
1288
1289         PMD_INIT_FUNC_TRACE();
1290         for (i = 0; i < dev->data->nb_queue_pairs; i++)
1291                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
1292         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
1293 }