drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_ether.h>
  50 #include <rte_malloc.h>
  51 #include <rte_launch.h>
  52 #include <rte_eal.h>
  53 #include <rte_per_lcore.h>
  54 #include <rte_lcore.h>
  55 #include <rte_atomic.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_ring.h>
  58 #include <rte_mempool.h>
  59 #include <rte_mbuf.h>
  60 #include <rte_string_fns.h>
  61 #include <rte_spinlock.h>
  62 #include <rte_hexdump.h>
  63
  64 #include "qat_logs.h"
  65 #include "qat_algs.h"
  66 #include "qat_crypto.h"
  67 #include "adf_transport_access_macros.h"
  68
  69 #define BYTE_LENGTH    8
  70
  71 static const struct rte_cryptodev_capabilities qat_pmd_capabilities[] = {
  72         {       /* SHA1 HMAC */
  73                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  74                 {.sym = {
  75                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  76                         {.auth = {
  77                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  78                                 .block_size = 64,
  79                                 .key_size = {
  80                                         .min = 64,
  81                                         .max = 64,
  82                                         .increment = 0
  83                                 },
  84                                 .digest_size = {
  85                                         .min = 20,
  86                                         .max = 20,
  87                                         .increment = 0
  88                                 },
  89                                 .aad_size = { 0 }
  90                         }, }
  91                 }, }
  92         },
  93         {       /* SHA256 HMAC */
  94                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  95                 {.sym = {
  96                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  97                         {.auth = {
  98                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
  99                                 .block_size = 64,
 100                                 .key_size = {
 101                                         .min = 64,
 102                                         .max = 64,
 103                                         .increment = 0
 104                                 },
 105                                 .digest_size = {
 106                                         .min = 32,
 107                                         .max = 32,
 108                                         .increment = 0
 109                                 },
 110                                 .aad_size = { 0 }
 111                         }, }
 112                 }, }
 113         },
 114         {       /* SHA512 HMAC */
 115                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 116                 {.sym = {
 117                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 118                         {.auth = {
 119                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 120                                 .block_size = 128,
 121                                 .key_size = {
 122                                         .min = 128,
 123                                         .max = 128,
 124                                         .increment = 0
 125                                 },
 126                                 .digest_size = {
 127                                         .min = 64,
 128                                         .max = 64,
 129                                         .increment = 0
 130                                 },
 131                                 .aad_size = { 0 }
 132                         }, }
 133                 }, }
 134         },
 135         {       /* AES XCBC MAC */
 136                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 137                 {.sym = {
 138                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 139                         {.auth = {
 140                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 141                                 .block_size = 16,
 142                                 .key_size = {
 143                                         .min = 16,
 144                                         .max = 16,
 145                                         .increment = 0
 146                                 },
 147                                 .digest_size = {
 148                                         .min = 16,
 149                                         .max = 16,
 150                                         .increment = 0
 151                                 },
 152                                 .aad_size = { 0 }
 153                         }, }
 154                 }, }
 155         },
 156         {       /* AES GCM (AUTH) */
 157                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 158                 {.sym = {
 159                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 160                         {.auth = {
 161                                 .algo = RTE_CRYPTO_AUTH_AES_GCM,
 162                                 .block_size = 16,
 163                                 .key_size = {
 164                                         .min = 16,
 165                                         .max = 32,
 166                                         .increment = 8
 167                                 },
 168                                 .digest_size = {
 169                                         .min = 8,
 170                                         .max = 16,
 171                                         .increment = 4
 172                                 },
 173                                 .aad_size = {
 174                                         .min = 8,
 175                                         .max = 12,
 176                                         .increment = 4
 177                                 }
 178                         }, }
 179                 }, }
 180         },
 181         {       /* SNOW3G (UIA2) */
 182                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 183                 {.sym = {
 184                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 185                         {.auth = {
 186                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 187                                 .block_size = 16,
 188                                 .key_size = {
 189                                         .min = 16,
 190                                         .max = 16,
 191                                         .increment = 0
 192                                 },
 193                                 .digest_size = {
 194                                         .min = 4,
 195                                         .max = 4,
 196                                         .increment = 0
 197                                 },
 198                                 .aad_size = {
 199                                         .min = 16,
 200                                         .max = 16,
 201                                         .increment = 0
 202                                 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* AES GCM (CIPHER) */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 210                         {.cipher = {
 211                                 .algo = RTE_CRYPTO_CIPHER_AES_GCM,
 212                                 .block_size = 16,
 213                                 .key_size = {
 214                                         .min = 16,
 215                                         .max = 32,
 216                                         .increment = 8
 217                                 },
 218                                 .iv_size = {
 219                                         .min = 16,
 220                                         .max = 16,
 221                                         .increment = 0
 222                                 }
 223                         }, }
 224                 }, }
 225         },
 226         {       /* AES CBC */
 227                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 228                 {.sym = {
 229                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 230                         {.cipher = {
 231                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 232                                 .block_size = 16,
 233                                 .key_size = {
 234                                         .min = 16,
 235                                         .max = 32,
 236                                         .increment = 8
 237                                 },
 238                                 .iv_size = {
 239                                         .min = 16,
 240                                         .max = 16,
 241                                         .increment = 0
 242                                 }
 243                         }, }
 244                 }, }
 245         },
 246         {       /* SNOW3G (UEA2) */
 247                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 248                 {.sym = {
 249                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 250                         {.cipher = {
 251                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 252                                 .block_size = 16,
 253                                 .key_size = {
 254                                         .min = 16,
 255                                         .max = 16,
 256                                         .increment = 0
 257                                 },
 258                                 .iv_size = {
 259                                         .min = 16,
 260                                         .max = 16,
 261                                         .increment = 0
 262                                 }
 263                         }, }
 264                 }, }
 265         },
 266         {       /* AES CTR */
 267                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 268                 {.sym = {
 269                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 270                         {.cipher = {
 271                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 272                                 .block_size = 16,
 273                                 .key_size = {
 274                                         .min = 16,
 275                                         .max = 32,
 276                                         .increment = 8
 277                                 },
 278                                 .iv_size = {
 279                                         .min = 16,
 280                                         .max = 16,
 281                                         .increment = 0
 282                                 }
 283                         }, }
 284                 }, }
 285         },
 286         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 287 };
 288
 289 static inline uint32_t
 290 adf_modulo(uint32_t data, uint32_t shift);
 291
 292 static inline int
 293 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg);
 294
 295 void qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 296                 void *session)
 297 {
 298         struct qat_session *sess = session;
 299         phys_addr_t cd_paddr;
 300
 301         PMD_INIT_FUNC_TRACE();
 302         if (session) {
 303                 cd_paddr = sess->cd_paddr;
 304                 memset(sess, 0, qat_crypto_sym_get_session_private_size(dev));
 305                 sess->cd_paddr = cd_paddr;
 306         } else
 307                 PMD_DRV_LOG(ERR, "NULL session");
 308 }
 309
 310 static int
 311 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 312 {
 313         /* Cipher Only */
 314         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 315                 return ICP_QAT_FW_LA_CMD_CIPHER;
 316
 317         /* Authentication Only */
 318         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 319                 return ICP_QAT_FW_LA_CMD_AUTH;
 320
 321         if (xform->next == NULL)
 322                 return -1;
 323
 324         /* Cipher then Authenticate */
 325         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 326                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 327                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 328
 329         /* Authenticate then Cipher */
 330         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 331                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 332                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 333
 334         return -1;
 335 }
 336
 337 static struct rte_crypto_auth_xform *
 338 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 339 {
 340         do {
 341                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 342                         return &xform->auth;
 343
 344                 xform = xform->next;
 345         } while (xform);
 346
 347         return NULL;
 348 }
 349
 350 static struct rte_crypto_cipher_xform *
 351 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 352 {
 353         do {
 354                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 355                         return &xform->cipher;
 356
 357                 xform = xform->next;
 358         } while (xform);
 359
 360         return NULL;
 361 }
 362 void *
 363 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 364                 struct rte_crypto_sym_xform *xform, void *session_private)
 365 {
 366         struct qat_pmd_private *internals = dev->data->dev_private;
 367
 368         struct qat_session *session = session_private;
 369
 370         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 371
 372         /* Get cipher xform from crypto xform chain */
 373         cipher_xform = qat_get_cipher_xform(xform);
 374
 375         switch (cipher_xform->algo) {
 376         case RTE_CRYPTO_CIPHER_AES_CBC:
 377                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 378                                 &session->qat_cipher_alg) != 0) {
 379                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 380                         goto error_out;
 381                 }
 382                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 383                 break;
 384         case RTE_CRYPTO_CIPHER_AES_GCM:
 385                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 386                                 &session->qat_cipher_alg) != 0) {
 387                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 388                         goto error_out;
 389                 }
 390                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 391                 break;
 392         case RTE_CRYPTO_CIPHER_AES_CTR:
 393                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 394                                 &session->qat_cipher_alg) != 0) {
 395                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 396                         goto error_out;
 397                 }
 398                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 399                 break;
 400         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 401                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 402                                         &session->qat_cipher_alg) != 0) {
 403                         PMD_DRV_LOG(ERR, "Invalid SNOW3G cipher key size");
 404                         goto error_out;
 405                 }
 406                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 407                 break;
 408         case RTE_CRYPTO_CIPHER_NULL:
 409         case RTE_CRYPTO_CIPHER_3DES_ECB:
 410         case RTE_CRYPTO_CIPHER_3DES_CBC:
 411         case RTE_CRYPTO_CIPHER_AES_ECB:
 412         case RTE_CRYPTO_CIPHER_AES_CCM:
 413         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 414                 PMD_DRV_LOG(ERR, "Crypto: Unsupported Cipher alg %u",
 415                                 cipher_xform->algo);
 416                 goto error_out;
 417         default:
 418                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 419                                 cipher_xform->algo);
 420                 goto error_out;
 421         }
 422
 423         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 424                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 425         else
 426                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 427
 428         if (qat_alg_aead_session_create_content_desc_cipher(session,
 429                                                 cipher_xform->key.data,
 430                                                 cipher_xform->key.length))
 431                 goto error_out;
 432
 433         return session;
 434
 435 error_out:
 436         rte_mempool_put(internals->sess_mp, session);
 437         return NULL;
 438 }
 439
 440
 441 void *
 442 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 443                 struct rte_crypto_sym_xform *xform, void *session_private)
 444 {
 445         struct qat_pmd_private *internals = dev->data->dev_private;
 446
 447         struct qat_session *session = session_private;
 448
 449         int qat_cmd_id;
 450
 451         PMD_INIT_FUNC_TRACE();
 452
 453         /* Get requested QAT command id */
 454         qat_cmd_id = qat_get_cmd_id(xform);
 455         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 456                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 457                 goto error_out;
 458         }
 459         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 460         switch (session->qat_cmd) {
 461         case ICP_QAT_FW_LA_CMD_CIPHER:
 462         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 463                 break;
 464         case ICP_QAT_FW_LA_CMD_AUTH:
 465         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 466                 break;
 467         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 468         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 469         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 470                 break;
 471         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 472         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 473         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 474                 break;
 475         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 476         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 477         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 478         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 479         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 480         case ICP_QAT_FW_LA_CMD_MGF1:
 481         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 482         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 483         case ICP_QAT_FW_LA_CMD_DELIMITER:
 484         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 485                 session->qat_cmd);
 486                 goto error_out;
 487         default:
 488         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 489                 session->qat_cmd);
 490                 goto error_out;
 491         }
 492         return session;
 493
 494 error_out:
 495         rte_mempool_put(internals->sess_mp, session);
 496         return NULL;
 497 }
 498
 499 struct qat_session *
 500 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 501                                 struct rte_crypto_sym_xform *xform,
 502                                 struct qat_session *session_private)
 503 {
 504
 505         struct qat_pmd_private *internals = dev->data->dev_private;
 506         struct qat_session *session = session_private;
 507         struct rte_crypto_auth_xform *auth_xform = NULL;
 508         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 509         auth_xform = qat_get_auth_xform(xform);
 510
 511         switch (auth_xform->algo) {
 512         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 513                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 514                 break;
 515         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 516                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 517                 break;
 518         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 519                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 520                 break;
 521         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 522                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 523                 break;
 524         case RTE_CRYPTO_AUTH_AES_GCM:
 525                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 526                 break;
 527         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 528                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 529                 break;
 530         case RTE_CRYPTO_AUTH_NULL:
 531         case RTE_CRYPTO_AUTH_SHA1:
 532         case RTE_CRYPTO_AUTH_SHA256:
 533         case RTE_CRYPTO_AUTH_SHA512:
 534         case RTE_CRYPTO_AUTH_SHA224:
 535         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 536         case RTE_CRYPTO_AUTH_SHA384:
 537         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 538         case RTE_CRYPTO_AUTH_MD5:
 539         case RTE_CRYPTO_AUTH_MD5_HMAC:
 540         case RTE_CRYPTO_AUTH_AES_CCM:
 541         case RTE_CRYPTO_AUTH_AES_GMAC:
 542         case RTE_CRYPTO_AUTH_KASUMI_F9:
 543         case RTE_CRYPTO_AUTH_AES_CMAC:
 544         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 545         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 546                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 547                                 auth_xform->algo);
 548                 goto error_out;
 549         default:
 550                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 551                                 auth_xform->algo);
 552                 goto error_out;
 553         }
 554         cipher_xform = qat_get_cipher_xform(xform);
 555
 556         if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
 557                         (session->qat_hash_alg ==
 558                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_64))  {
 559                 if (qat_alg_aead_session_create_content_desc_auth(session,
 560                                 cipher_xform->key.data,
 561                                 cipher_xform->key.length,
 562                                 auth_xform->add_auth_data_length,
 563                                 auth_xform->digest_length,
 564                                 auth_xform->op))
 565                         goto error_out;
 566         } else {
 567                 if (qat_alg_aead_session_create_content_desc_auth(session,
 568                                 auth_xform->key.data,
 569                                 auth_xform->key.length,
 570                                 auth_xform->add_auth_data_length,
 571                                 auth_xform->digest_length,
 572                                 auth_xform->op))
 573                         goto error_out;
 574         }
 575         return session;
 576
 577 error_out:
 578         rte_mempool_put(internals->sess_mp, session);
 579         return NULL;
 580 }
 581
 582 unsigned qat_crypto_sym_get_session_private_size(
 583                 struct rte_cryptodev *dev __rte_unused)
 584 {
 585         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 586 }
 587
 588
 589 uint16_t
 590 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 591                 uint16_t nb_ops)
 592 {
 593         register struct qat_queue *queue;
 594         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 595         register uint32_t nb_ops_sent = 0;
 596         register struct rte_crypto_op **cur_op = ops;
 597         register int ret;
 598         uint16_t nb_ops_possible = nb_ops;
 599         register uint8_t *base_addr;
 600         register uint32_t tail;
 601         int overflow;
 602
 603         if (unlikely(nb_ops == 0))
 604                 return 0;
 605
 606         /* read params used a lot in main loop into registers */
 607         queue = &(tmp_qp->tx_q);
 608         base_addr = (uint8_t *)queue->base_addr;
 609         tail = queue->tail;
 610
 611         /* Find how many can actually fit on the ring */
 612         overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops)
 613                                 - queue->max_inflights;
 614         if (overflow > 0) {
 615                 rte_atomic16_sub(&tmp_qp->inflights16, overflow);
 616                 nb_ops_possible = nb_ops - overflow;
 617                 if (nb_ops_possible == 0)
 618                         return 0;
 619         }
 620
 621         while (nb_ops_sent != nb_ops_possible) {
 622                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail);
 623                 if (ret != 0) {
 624                         tmp_qp->stats.enqueue_err_count++;
 625                         if (nb_ops_sent == 0)
 626                                 return 0;
 627                         goto kick_tail;
 628                 }
 629
 630                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 631                 nb_ops_sent++;
 632                 cur_op++;
 633         }
 634 kick_tail:
 635         WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number,
 636                         queue->hw_queue_number, tail);
 637         queue->tail = tail;
 638         tmp_qp->stats.enqueued_count += nb_ops_sent;
 639         return nb_ops_sent;
 640 }
 641
 642 uint16_t
 643 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 644                 uint16_t nb_ops)
 645 {
 646         struct qat_queue *queue;
 647         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 648         uint32_t msg_counter = 0;
 649         struct rte_crypto_op *rx_op;
 650         struct icp_qat_fw_comn_resp *resp_msg;
 651
 652         queue = &(tmp_qp->rx_q);
 653         resp_msg = (struct icp_qat_fw_comn_resp *)
 654                         ((uint8_t *)queue->base_addr + queue->head);
 655
 656         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 657                         msg_counter != nb_ops) {
 658                 rx_op = (struct rte_crypto_op *)(uintptr_t)
 659                                 (resp_msg->opaque_data);
 660
 661 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 662                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
 663                                 sizeof(struct icp_qat_fw_comn_resp));
 664 #endif
 665                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
 666                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
 667                                         resp_msg->comn_hdr.comn_status)) {
 668                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 669                 } else {
 670                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 671                 }
 672                 *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG;
 673                 queue->head = adf_modulo(queue->head +
 674                                 queue->msg_size,
 675                                 ADF_RING_SIZE_MODULO(queue->queue_size));
 676                 resp_msg = (struct icp_qat_fw_comn_resp *)
 677                                         ((uint8_t *)queue->base_addr +
 678                                                         queue->head);
 679                 *ops = rx_op;
 680                 ops++;
 681                 msg_counter++;
 682         }
 683         if (msg_counter > 0) {
 684                 WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr,
 685                                         queue->hw_bundle_number,
 686                                         queue->hw_queue_number, queue->head);
 687                 rte_atomic16_sub(&tmp_qp->inflights16, msg_counter);
 688                 tmp_qp->stats.dequeued_count += msg_counter;
 689         }
 690         return msg_counter;
 691 }
 692
 693 static inline int
 694 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg)
 695 {
 696         struct qat_session *ctx;
 697         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 698         struct icp_qat_fw_la_auth_req_params *auth_param;
 699         register struct icp_qat_fw_la_bulk_req *qat_req;
 700
 701 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 702         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 703                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
 704                                 "operation requests, op (%p) is not a "
 705                                 "symmetric operation.", op);
 706                 return -EINVAL;
 707         }
 708 #endif
 709         if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) {
 710                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
 711                                 " requests, op (%p) is sessionless.", op);
 712                 return -EINVAL;
 713         }
 714
 715         if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) {
 716                 PMD_DRV_LOG(ERR, "Session was not created for this device");
 717                 return -EINVAL;
 718         }
 719
 720         ctx = (struct qat_session *)op->sym->session->_private;
 721         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 722         *qat_req = ctx->fw_req;
 723         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
 724
 725         qat_req->comn_mid.dst_length =
 726                 qat_req->comn_mid.src_length =
 727                                 rte_pktmbuf_data_len(op->sym->m_src);
 728
 729         qat_req->comn_mid.dest_data_addr =
 730                 qat_req->comn_mid.src_data_addr =
 731                             rte_pktmbuf_mtophys(op->sym->m_src);
 732
 733         if (unlikely(op->sym->m_dst != NULL)) {
 734                 qat_req->comn_mid.dest_data_addr =
 735                                 rte_pktmbuf_mtophys(op->sym->m_dst);
 736                 qat_req->comn_mid.dst_length =
 737                                 rte_pktmbuf_data_len(op->sym->m_dst);
 738         }
 739
 740         cipher_param = (void *)&qat_req->serv_specif_rqpars;
 741         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 742
 743         cipher_param->cipher_length = op->sym->cipher.data.length;
 744         cipher_param->cipher_offset = op->sym->cipher.data.offset;
 745         if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
 746                 if (unlikely((cipher_param->cipher_length % BYTE_LENGTH != 0) ||
 747                                 (cipher_param->cipher_offset
 748                                         % BYTE_LENGTH != 0))) {
 749                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 750                                 "supports byte aligned values");
 751                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 752                         return -EINVAL;
 753                 }
 754                 cipher_param->cipher_length >>= 3;
 755                 cipher_param->cipher_offset >>= 3;
 756         }
 757
 758         if (op->sym->cipher.iv.length && (op->sym->cipher.iv.length <=
 759                         sizeof(cipher_param->u.cipher_IV_array))) {
 760                 rte_memcpy(cipher_param->u.cipher_IV_array,
 761                                 op->sym->cipher.iv.data,
 762                                 op->sym->cipher.iv.length);
 763         } else {
 764                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
 765                                 qat_req->comn_hdr.serv_specif_flags,
 766                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
 767                 cipher_param->u.s.cipher_IV_ptr = op->sym->cipher.iv.phys_addr;
 768         }
 769         if (op->sym->auth.digest.phys_addr) {
 770                 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
 771                                 qat_req->comn_hdr.serv_specif_flags,
 772                                 ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
 773                 auth_param->auth_res_addr = op->sym->auth.digest.phys_addr;
 774         }
 775         auth_param->auth_off = op->sym->auth.data.offset;
 776         auth_param->auth_len = op->sym->auth.data.length;
 777         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) {
 778                 if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) ||
 779                                 (auth_param->auth_len % BYTE_LENGTH != 0))) {
 780                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 781                                 "supports byte aligned values");
 782                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 783                         return -EINVAL;
 784                 }
 785                 auth_param->auth_off >>= 3;
 786                 auth_param->auth_len >>= 3;
 787         }
 788         auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr;
 789         /* (GCM) aad length(240 max) will be at this location after precompute */
 790         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 791                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
 792                 struct icp_qat_hw_auth_algo_blk *hash;
 793
 794                 if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER)
 795                         hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&ctx->cd);
 796                 else
 797                         hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&ctx->cd +
 798                                 sizeof(struct icp_qat_hw_cipher_algo_blk));
 799
 800                 auth_param->u2.aad_sz = ALIGN_POW2_ROUNDUP(hash->sha.state1[
 801                                         ICP_QAT_HW_GALOIS_128_STATE1_SZ +
 802                                         ICP_QAT_HW_GALOIS_H_SZ + 3], 16);
 803                 if (op->sym->cipher.iv.length == 12) {
 804                         /*
 805                          * For GCM a 12 bit IV is allowed,
 806                          * but we need to inform the f/w
 807                          */
 808                         ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 809                                 qat_req->comn_hdr.serv_specif_flags,
 810                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 811                 }
 812         }
 813         auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3;
 814
 815
 816 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 817         rte_hexdump(stdout, "qat_req:", qat_req,
 818                         sizeof(struct icp_qat_fw_la_bulk_req));
 819         rte_hexdump(stdout, "src_data:",
 820                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
 821                         rte_pktmbuf_data_len(op->sym->m_src));
 822         rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data,
 823                         op->sym->cipher.iv.length);
 824         rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
 825                         op->sym->auth.digest.length);
 826         rte_hexdump(stdout, "aad:", op->sym->auth.aad.data,
 827                         op->sym->auth.aad.length);
 828 #endif
 829         return 0;
 830 }
 831
 832 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
 833 {
 834         uint32_t div = data >> shift;
 835         uint32_t mult = div << shift;
 836
 837         return data - mult;
 838 }
 839
 840 void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess)
 841 {
 842         struct rte_cryptodev_sym_session *sess = sym_sess;
 843         struct qat_session *s = (void *)sess->_private;
 844
 845         PMD_INIT_FUNC_TRACE();
 846         s->cd_paddr = rte_mempool_virt2phy(mp, sess) +
 847                 offsetof(struct qat_session, cd) +
 848                 offsetof(struct rte_cryptodev_sym_session, _private);
 849 }
 850
 851 int qat_dev_config(__rte_unused struct rte_cryptodev *dev)
 852 {
 853         PMD_INIT_FUNC_TRACE();
 854         return -ENOTSUP;
 855 }
 856
 857 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
 858 {
 859         PMD_INIT_FUNC_TRACE();
 860         return 0;
 861 }
 862
 863 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
 864 {
 865         PMD_INIT_FUNC_TRACE();
 866 }
 867
 868 int qat_dev_close(struct rte_cryptodev *dev)
 869 {
 870         int i, ret;
 871
 872         PMD_INIT_FUNC_TRACE();
 873
 874         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 875                 ret = qat_crypto_sym_qp_release(dev, i);
 876                 if (ret < 0)
 877                         return ret;
 878         }
 879
 880         return 0;
 881 }
 882
 883 void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev,
 884                                 struct rte_cryptodev_info *info)
 885 {
 886         struct qat_pmd_private *internals = dev->data->dev_private;
 887
 888         PMD_INIT_FUNC_TRACE();
 889         if (info != NULL) {
 890                 info->max_nb_queue_pairs =
 891                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
 892                                 ADF_NUM_BUNDLES_PER_DEV;
 893                 info->feature_flags = dev->feature_flags;
 894                 info->capabilities = qat_pmd_capabilities;
 895                 info->sym.max_nb_sessions = internals->max_nb_sessions;
 896                 info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD;
 897         }
 898 }
 899
 900 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
 901                 struct rte_cryptodev_stats *stats)
 902 {
 903         int i;
 904         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
 905
 906         PMD_INIT_FUNC_TRACE();
 907         if (stats == NULL) {
 908                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
 909                 return;
 910         }
 911         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 912                 if (qp[i] == NULL) {
 913                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
 914                         continue;
 915                 }
 916
 917                 stats->enqueued_count += qp[i]->stats.enqueued_count;
 918                 stats->dequeued_count += qp[i]->stats.enqueued_count;
 919                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
 920                 stats->dequeue_err_count += qp[i]->stats.enqueue_err_count;
 921         }
 922 }
 923
 924 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
 925 {
 926         int i;
 927         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
 928
 929         PMD_INIT_FUNC_TRACE();
 930         for (i = 0; i < dev->data->nb_queue_pairs; i++)
 931                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
 932         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
 933 }