drivers/crypto/qat/qat_crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2015-2016 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *       * Redistributions of source code must retain the above copyright
  12  *         notice, this list of conditions and the following disclaimer.
  13  *       * Redistributions in binary form must reproduce the above copyright
  14  *         notice, this list of conditions and the following disclaimer in
  15  *         the documentation and/or other materials provided with the
  16  *         distribution.
  17  *       * Neither the name of Intel Corporation nor the names of its
  18  *         contributors may be used to endorse or promote products derived
  19  *         from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_tailq.h>
  49 #include <rte_ether.h>
  50 #include <rte_malloc.h>
  51 #include <rte_launch.h>
  52 #include <rte_eal.h>
  53 #include <rte_per_lcore.h>
  54 #include <rte_lcore.h>
  55 #include <rte_atomic.h>
  56 #include <rte_branch_prediction.h>
  57 #include <rte_ring.h>
  58 #include <rte_mempool.h>
  59 #include <rte_mbuf.h>
  60 #include <rte_string_fns.h>
  61 #include <rte_spinlock.h>
  62 #include <rte_hexdump.h>
  63
  64 #include "qat_logs.h"
  65 #include "qat_algs.h"
  66 #include "qat_crypto.h"
  67 #include "adf_transport_access_macros.h"
  68
  69 #define BYTE_LENGTH    8
  70
  71 static const struct rte_cryptodev_capabilities qat_pmd_capabilities[] = {
  72         {       /* SHA1 HMAC */
  73                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  74                 {.sym = {
  75                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  76                         {.auth = {
  77                                 .algo = RTE_CRYPTO_AUTH_SHA1_HMAC,
  78                                 .block_size = 64,
  79                                 .key_size = {
  80                                         .min = 64,
  81                                         .max = 64,
  82                                         .increment = 0
  83                                 },
  84                                 .digest_size = {
  85                                         .min = 20,
  86                                         .max = 20,
  87                                         .increment = 0
  88                                 },
  89                                 .aad_size = { 0 }
  90                         }, }
  91                 }, }
  92         },
  93         {       /* SHA256 HMAC */
  94                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
  95                 {.sym = {
  96                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
  97                         {.auth = {
  98                                 .algo = RTE_CRYPTO_AUTH_SHA256_HMAC,
  99                                 .block_size = 64,
 100                                 .key_size = {
 101                                         .min = 64,
 102                                         .max = 64,
 103                                         .increment = 0
 104                                 },
 105                                 .digest_size = {
 106                                         .min = 32,
 107                                         .max = 32,
 108                                         .increment = 0
 109                                 },
 110                                 .aad_size = { 0 }
 111                         }, }
 112                 }, }
 113         },
 114         {       /* SHA512 HMAC */
 115                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 116                 {.sym = {
 117                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 118                         {.auth = {
 119                                 .algo = RTE_CRYPTO_AUTH_SHA512_HMAC,
 120                                 .block_size = 128,
 121                                 .key_size = {
 122                                         .min = 128,
 123                                         .max = 128,
 124                                         .increment = 0
 125                                 },
 126                                 .digest_size = {
 127                                         .min = 64,
 128                                         .max = 64,
 129                                         .increment = 0
 130                                 },
 131                                 .aad_size = { 0 }
 132                         }, }
 133                 }, }
 134         },
 135         {       /* AES XCBC MAC */
 136                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 137                 {.sym = {
 138                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 139                         {.auth = {
 140                                 .algo = RTE_CRYPTO_AUTH_AES_XCBC_MAC,
 141                                 .block_size = 16,
 142                                 .key_size = {
 143                                         .min = 16,
 144                                         .max = 16,
 145                                         .increment = 0
 146                                 },
 147                                 .digest_size = {
 148                                         .min = 16,
 149                                         .max = 16,
 150                                         .increment = 0
 151                                 },
 152                                 .aad_size = { 0 }
 153                         }, }
 154                 }, }
 155         },
 156         {       /* AES GCM (AUTH) */
 157                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 158                 {.sym = {
 159                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 160                         {.auth = {
 161                                 .algo = RTE_CRYPTO_AUTH_AES_GCM,
 162                                 .block_size = 16,
 163                                 .key_size = {
 164                                         .min = 16,
 165                                         .max = 32,
 166                                         .increment = 8
 167                                 },
 168                                 .digest_size = {
 169                                         .min = 8,
 170                                         .max = 16,
 171                                         .increment = 4
 172                                 },
 173                                 .aad_size = {
 174                                         .min = 8,
 175                                         .max = 12,
 176                                         .increment = 4
 177                                 }
 178                         }, }
 179                 }, }
 180         },
 181         {       /* SNOW3G (UIA2) */
 182                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 183                 {.sym = {
 184                         .xform_type = RTE_CRYPTO_SYM_XFORM_AUTH,
 185                         {.auth = {
 186                                 .algo = RTE_CRYPTO_AUTH_SNOW3G_UIA2,
 187                                 .block_size = 16,
 188                                 .key_size = {
 189                                         .min = 16,
 190                                         .max = 16,
 191                                         .increment = 0
 192                                 },
 193                                 .digest_size = {
 194                                         .min = 4,
 195                                         .max = 4,
 196                                         .increment = 0
 197                                 },
 198                                 .aad_size = {
 199                                         .min = 16,
 200                                         .max = 16,
 201                                         .increment = 0
 202                                 }
 203                         }, }
 204                 }, }
 205         },
 206         {       /* AES GCM (CIPHER) */
 207                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 208                 {.sym = {
 209                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 210                         {.cipher = {
 211                                 .algo = RTE_CRYPTO_CIPHER_AES_GCM,
 212                                 .block_size = 16,
 213                                 .key_size = {
 214                                         .min = 16,
 215                                         .max = 32,
 216                                         .increment = 8
 217                                 },
 218                                 .iv_size = {
 219                                         .min = 16,
 220                                         .max = 16,
 221                                         .increment = 0
 222                                 }
 223                         }, }
 224                 }, }
 225         },
 226         {       /* AES CBC */
 227                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 228                 {.sym = {
 229                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 230                         {.cipher = {
 231                                 .algo = RTE_CRYPTO_CIPHER_AES_CBC,
 232                                 .block_size = 16,
 233                                 .key_size = {
 234                                         .min = 16,
 235                                         .max = 32,
 236                                         .increment = 8
 237                                 },
 238                                 .iv_size = {
 239                                         .min = 16,
 240                                         .max = 16,
 241                                         .increment = 0
 242                                 }
 243                         }, }
 244                 }, }
 245         },
 246         {       /* SNOW3G (UEA2) */
 247                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 248                 {.sym = {
 249                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 250                         {.cipher = {
 251                                 .algo = RTE_CRYPTO_CIPHER_SNOW3G_UEA2,
 252                                 .block_size = 16,
 253                                 .key_size = {
 254                                         .min = 16,
 255                                         .max = 16,
 256                                         .increment = 0
 257                                 },
 258                                 .iv_size = {
 259                                         .min = 16,
 260                                         .max = 16,
 261                                         .increment = 0
 262                                 }
 263                         }, }
 264                 }, }
 265         },
 266         {       /* AES CTR */
 267                 .op = RTE_CRYPTO_OP_TYPE_SYMMETRIC,
 268                 {.sym = {
 269                         .xform_type = RTE_CRYPTO_SYM_XFORM_CIPHER,
 270                         {.cipher = {
 271                                 .algo = RTE_CRYPTO_CIPHER_AES_CTR,
 272                                 .block_size = 16,
 273                                 .key_size = {
 274                                         .min = 16,
 275                                         .max = 32,
 276                                         .increment = 8
 277                                 },
 278                                 .iv_size = {
 279                                         .min = 16,
 280                                         .max = 16,
 281                                         .increment = 0
 282                                 }
 283                         }, }
 284                 }, }
 285         },
 286         RTE_CRYPTODEV_END_OF_CAPABILITIES_LIST()
 287 };
 288
 289 static inline uint32_t
 290 adf_modulo(uint32_t data, uint32_t shift);
 291
 292 static inline int
 293 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg);
 294
 295 void qat_crypto_sym_clear_session(struct rte_cryptodev *dev,
 296                 void *session)
 297 {
 298         struct qat_session *sess = session;
 299         phys_addr_t cd_paddr;
 300
 301         PMD_INIT_FUNC_TRACE();
 302         if (session) {
 303                 cd_paddr = sess->cd_paddr;
 304                 memset(sess, 0, qat_crypto_sym_get_session_private_size(dev));
 305                 sess->cd_paddr = cd_paddr;
 306         } else
 307                 PMD_DRV_LOG(ERR, "NULL session");
 308 }
 309
 310 static int
 311 qat_get_cmd_id(const struct rte_crypto_sym_xform *xform)
 312 {
 313         /* Cipher Only */
 314         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER && xform->next == NULL)
 315                 return ICP_QAT_FW_LA_CMD_CIPHER;
 316
 317         /* Authentication Only */
 318         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH && xform->next == NULL)
 319                 return ICP_QAT_FW_LA_CMD_AUTH;
 320
 321         if (xform->next == NULL)
 322                 return -1;
 323
 324         /* Cipher then Authenticate */
 325         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER &&
 326                         xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 327                 return ICP_QAT_FW_LA_CMD_CIPHER_HASH;
 328
 329         /* Authenticate then Cipher */
 330         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH &&
 331                         xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 332                 return ICP_QAT_FW_LA_CMD_HASH_CIPHER;
 333
 334         return -1;
 335 }
 336
 337 static struct rte_crypto_auth_xform *
 338 qat_get_auth_xform(struct rte_crypto_sym_xform *xform)
 339 {
 340         do {
 341                 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 342                         return &xform->auth;
 343
 344                 xform = xform->next;
 345         } while (xform);
 346
 347         return NULL;
 348 }
 349
 350 static struct rte_crypto_cipher_xform *
 351 qat_get_cipher_xform(struct rte_crypto_sym_xform *xform)
 352 {
 353         do {
 354                 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 355                         return &xform->cipher;
 356
 357                 xform = xform->next;
 358         } while (xform);
 359
 360         return NULL;
 361 }
 362 void *
 363 qat_crypto_sym_configure_session_cipher(struct rte_cryptodev *dev,
 364                 struct rte_crypto_sym_xform *xform, void *session_private)
 365 {
 366         struct qat_pmd_private *internals = dev->data->dev_private;
 367
 368         struct qat_session *session = session_private;
 369
 370         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 371
 372         /* Get cipher xform from crypto xform chain */
 373         cipher_xform = qat_get_cipher_xform(xform);
 374
 375         switch (cipher_xform->algo) {
 376         case RTE_CRYPTO_CIPHER_AES_CBC:
 377                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 378                                 &session->qat_cipher_alg) != 0) {
 379                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 380                         goto error_out;
 381                 }
 382                 session->qat_mode = ICP_QAT_HW_CIPHER_CBC_MODE;
 383                 break;
 384         case RTE_CRYPTO_CIPHER_AES_GCM:
 385                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 386                                 &session->qat_cipher_alg) != 0) {
 387                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 388                         goto error_out;
 389                 }
 390                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 391                 break;
 392         case RTE_CRYPTO_CIPHER_AES_CTR:
 393                 if (qat_alg_validate_aes_key(cipher_xform->key.length,
 394                                 &session->qat_cipher_alg) != 0) {
 395                         PMD_DRV_LOG(ERR, "Invalid AES cipher key size");
 396                         goto error_out;
 397                 }
 398                 session->qat_mode = ICP_QAT_HW_CIPHER_CTR_MODE;
 399                 break;
 400         case RTE_CRYPTO_CIPHER_SNOW3G_UEA2:
 401                 if (qat_alg_validate_snow3g_key(cipher_xform->key.length,
 402                                         &session->qat_cipher_alg) != 0) {
 403                         PMD_DRV_LOG(ERR, "Invalid SNOW3G cipher key size");
 404                         goto error_out;
 405                 }
 406                 session->qat_mode = ICP_QAT_HW_CIPHER_ECB_MODE;
 407                 break;
 408         case RTE_CRYPTO_CIPHER_NULL:
 409         case RTE_CRYPTO_CIPHER_3DES_ECB:
 410         case RTE_CRYPTO_CIPHER_3DES_CBC:
 411         case RTE_CRYPTO_CIPHER_AES_ECB:
 412         case RTE_CRYPTO_CIPHER_AES_CCM:
 413         case RTE_CRYPTO_CIPHER_KASUMI_F8:
 414                 PMD_DRV_LOG(ERR, "Crypto: Unsupported Cipher alg %u",
 415                                 cipher_xform->algo);
 416                 goto error_out;
 417         default:
 418                 PMD_DRV_LOG(ERR, "Crypto: Undefined Cipher specified %u\n",
 419                                 cipher_xform->algo);
 420                 goto error_out;
 421         }
 422
 423         if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)
 424                 session->qat_dir = ICP_QAT_HW_CIPHER_ENCRYPT;
 425         else
 426                 session->qat_dir = ICP_QAT_HW_CIPHER_DECRYPT;
 427
 428         if (qat_alg_aead_session_create_content_desc_cipher(session,
 429                                                 cipher_xform->key.data,
 430                                                 cipher_xform->key.length))
 431                 goto error_out;
 432
 433         return session;
 434
 435 error_out:
 436         rte_mempool_put(internals->sess_mp, session);
 437         return NULL;
 438 }
 439
 440
 441 void *
 442 qat_crypto_sym_configure_session(struct rte_cryptodev *dev,
 443                 struct rte_crypto_sym_xform *xform, void *session_private)
 444 {
 445         struct qat_pmd_private *internals = dev->data->dev_private;
 446
 447         struct qat_session *session = session_private;
 448
 449         int qat_cmd_id;
 450
 451         PMD_INIT_FUNC_TRACE();
 452
 453         /* Get requested QAT command id */
 454         qat_cmd_id = qat_get_cmd_id(xform);
 455         if (qat_cmd_id < 0 || qat_cmd_id >= ICP_QAT_FW_LA_CMD_DELIMITER) {
 456                 PMD_DRV_LOG(ERR, "Unsupported xform chain requested");
 457                 goto error_out;
 458         }
 459         session->qat_cmd = (enum icp_qat_fw_la_cmd_id)qat_cmd_id;
 460         switch (session->qat_cmd) {
 461         case ICP_QAT_FW_LA_CMD_CIPHER:
 462         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 463                 break;
 464         case ICP_QAT_FW_LA_CMD_AUTH:
 465         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 466                 break;
 467         case ICP_QAT_FW_LA_CMD_CIPHER_HASH:
 468         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 469         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 470                 break;
 471         case ICP_QAT_FW_LA_CMD_HASH_CIPHER:
 472         session = qat_crypto_sym_configure_session_auth(dev, xform, session);
 473         session = qat_crypto_sym_configure_session_cipher(dev, xform, session);
 474                 break;
 475         case ICP_QAT_FW_LA_CMD_TRNG_GET_RANDOM:
 476         case ICP_QAT_FW_LA_CMD_TRNG_TEST:
 477         case ICP_QAT_FW_LA_CMD_SSL3_KEY_DERIVE:
 478         case ICP_QAT_FW_LA_CMD_TLS_V1_1_KEY_DERIVE:
 479         case ICP_QAT_FW_LA_CMD_TLS_V1_2_KEY_DERIVE:
 480         case ICP_QAT_FW_LA_CMD_MGF1:
 481         case ICP_QAT_FW_LA_CMD_AUTH_PRE_COMP:
 482         case ICP_QAT_FW_LA_CMD_CIPHER_PRE_COMP:
 483         case ICP_QAT_FW_LA_CMD_DELIMITER:
 484         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 485                 session->qat_cmd);
 486                 goto error_out;
 487         default:
 488         PMD_DRV_LOG(ERR, "Unsupported Service %u",
 489                 session->qat_cmd);
 490                 goto error_out;
 491         }
 492         return session;
 493
 494 error_out:
 495         rte_mempool_put(internals->sess_mp, session);
 496         return NULL;
 497 }
 498
 499 struct qat_session *
 500 qat_crypto_sym_configure_session_auth(struct rte_cryptodev *dev,
 501                                 struct rte_crypto_sym_xform *xform,
 502                                 struct qat_session *session_private)
 503 {
 504
 505         struct qat_pmd_private *internals = dev->data->dev_private;
 506         struct qat_session *session = session_private;
 507         struct rte_crypto_auth_xform *auth_xform = NULL;
 508         struct rte_crypto_cipher_xform *cipher_xform = NULL;
 509         auth_xform = qat_get_auth_xform(xform);
 510
 511         switch (auth_xform->algo) {
 512         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 513                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA1;
 514                 break;
 515         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 516                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA256;
 517                 break;
 518         case RTE_CRYPTO_AUTH_SHA512_HMAC:
 519                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SHA512;
 520                 break;
 521         case RTE_CRYPTO_AUTH_AES_XCBC_MAC:
 522                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_AES_XCBC_MAC;
 523                 break;
 524         case RTE_CRYPTO_AUTH_AES_GCM:
 525                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_GALOIS_128;
 526                 break;
 527         case RTE_CRYPTO_AUTH_SNOW3G_UIA2:
 528                 session->qat_hash_alg = ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2;
 529                 break;
 530         case RTE_CRYPTO_AUTH_NULL:
 531         case RTE_CRYPTO_AUTH_SHA1:
 532         case RTE_CRYPTO_AUTH_SHA256:
 533         case RTE_CRYPTO_AUTH_SHA512:
 534         case RTE_CRYPTO_AUTH_SHA224:
 535         case RTE_CRYPTO_AUTH_SHA224_HMAC:
 536         case RTE_CRYPTO_AUTH_SHA384:
 537         case RTE_CRYPTO_AUTH_SHA384_HMAC:
 538         case RTE_CRYPTO_AUTH_MD5:
 539         case RTE_CRYPTO_AUTH_MD5_HMAC:
 540         case RTE_CRYPTO_AUTH_AES_CCM:
 541         case RTE_CRYPTO_AUTH_AES_GMAC:
 542         case RTE_CRYPTO_AUTH_KASUMI_F9:
 543         case RTE_CRYPTO_AUTH_AES_CMAC:
 544         case RTE_CRYPTO_AUTH_AES_CBC_MAC:
 545         case RTE_CRYPTO_AUTH_ZUC_EIA3:
 546                 PMD_DRV_LOG(ERR, "Crypto: Unsupported hash alg %u",
 547                                 auth_xform->algo);
 548                 goto error_out;
 549         default:
 550                 PMD_DRV_LOG(ERR, "Crypto: Undefined Hash algo %u specified",
 551                                 auth_xform->algo);
 552                 goto error_out;
 553         }
 554         cipher_xform = qat_get_cipher_xform(xform);
 555
 556         if ((session->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128) ||
 557                         (session->qat_hash_alg ==
 558                                 ICP_QAT_HW_AUTH_ALGO_GALOIS_64))  {
 559                 if (qat_alg_aead_session_create_content_desc_auth(session,
 560                                 cipher_xform->key.data,
 561                                 cipher_xform->key.length,
 562                                 auth_xform->add_auth_data_length,
 563                                 auth_xform->digest_length))
 564                         goto error_out;
 565         } else {
 566                 if (qat_alg_aead_session_create_content_desc_auth(session,
 567                                 auth_xform->key.data,
 568                                 auth_xform->key.length,
 569                                 auth_xform->add_auth_data_length,
 570                                 auth_xform->digest_length))
 571                         goto error_out;
 572         }
 573         return session;
 574
 575 error_out:
 576         rte_mempool_put(internals->sess_mp, session);
 577         return NULL;
 578 }
 579
 580 unsigned qat_crypto_sym_get_session_private_size(
 581                 struct rte_cryptodev *dev __rte_unused)
 582 {
 583         return RTE_ALIGN_CEIL(sizeof(struct qat_session), 8);
 584 }
 585
 586
 587 uint16_t
 588 qat_pmd_enqueue_op_burst(void *qp, struct rte_crypto_op **ops,
 589                 uint16_t nb_ops)
 590 {
 591         register struct qat_queue *queue;
 592         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 593         register uint32_t nb_ops_sent = 0;
 594         register struct rte_crypto_op **cur_op = ops;
 595         register int ret;
 596         uint16_t nb_ops_possible = nb_ops;
 597         register uint8_t *base_addr;
 598         register uint32_t tail;
 599         int overflow;
 600
 601         if (unlikely(nb_ops == 0))
 602                 return 0;
 603
 604         /* read params used a lot in main loop into registers */
 605         queue = &(tmp_qp->tx_q);
 606         base_addr = (uint8_t *)queue->base_addr;
 607         tail = queue->tail;
 608
 609         /* Find how many can actually fit on the ring */
 610         overflow = rte_atomic16_add_return(&tmp_qp->inflights16, nb_ops)
 611                                 - queue->max_inflights;
 612         if (overflow > 0) {
 613                 rte_atomic16_sub(&tmp_qp->inflights16, overflow);
 614                 nb_ops_possible = nb_ops - overflow;
 615                 if (nb_ops_possible == 0)
 616                         return 0;
 617         }
 618
 619         while (nb_ops_sent != nb_ops_possible) {
 620                 ret = qat_write_hw_desc_entry(*cur_op, base_addr + tail);
 621                 if (ret != 0) {
 622                         tmp_qp->stats.enqueue_err_count++;
 623                         if (nb_ops_sent == 0)
 624                                 return 0;
 625                         goto kick_tail;
 626                 }
 627
 628                 tail = adf_modulo(tail + queue->msg_size, queue->modulo);
 629                 nb_ops_sent++;
 630                 cur_op++;
 631         }
 632 kick_tail:
 633         WRITE_CSR_RING_TAIL(tmp_qp->mmap_bar_addr, queue->hw_bundle_number,
 634                         queue->hw_queue_number, tail);
 635         queue->tail = tail;
 636         tmp_qp->stats.enqueued_count += nb_ops_sent;
 637         return nb_ops_sent;
 638 }
 639
 640 uint16_t
 641 qat_pmd_dequeue_op_burst(void *qp, struct rte_crypto_op **ops,
 642                 uint16_t nb_ops)
 643 {
 644         struct qat_queue *queue;
 645         struct qat_qp *tmp_qp = (struct qat_qp *)qp;
 646         uint32_t msg_counter = 0;
 647         struct rte_crypto_op *rx_op;
 648         struct icp_qat_fw_comn_resp *resp_msg;
 649
 650         queue = &(tmp_qp->rx_q);
 651         resp_msg = (struct icp_qat_fw_comn_resp *)
 652                         ((uint8_t *)queue->base_addr + queue->head);
 653
 654         while (*(uint32_t *)resp_msg != ADF_RING_EMPTY_SIG &&
 655                         msg_counter != nb_ops) {
 656                 rx_op = (struct rte_crypto_op *)(uintptr_t)
 657                                 (resp_msg->opaque_data);
 658
 659 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_RX
 660                 rte_hexdump(stdout, "qat_response:", (uint8_t *)resp_msg,
 661                                 sizeof(struct icp_qat_fw_comn_resp));
 662 #endif
 663                 if (ICP_QAT_FW_COMN_STATUS_FLAG_OK !=
 664                                 ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(
 665                                         resp_msg->comn_hdr.comn_status)) {
 666                         rx_op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 667                 } else {
 668                         rx_op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 669                 }
 670                 *(uint32_t *)resp_msg = ADF_RING_EMPTY_SIG;
 671                 queue->head = adf_modulo(queue->head +
 672                                 queue->msg_size,
 673                                 ADF_RING_SIZE_MODULO(queue->queue_size));
 674                 resp_msg = (struct icp_qat_fw_comn_resp *)
 675                                         ((uint8_t *)queue->base_addr +
 676                                                         queue->head);
 677                 *ops = rx_op;
 678                 ops++;
 679                 msg_counter++;
 680         }
 681         if (msg_counter > 0) {
 682                 WRITE_CSR_RING_HEAD(tmp_qp->mmap_bar_addr,
 683                                         queue->hw_bundle_number,
 684                                         queue->hw_queue_number, queue->head);
 685                 rte_atomic16_sub(&tmp_qp->inflights16, msg_counter);
 686                 tmp_qp->stats.dequeued_count += msg_counter;
 687         }
 688         return msg_counter;
 689 }
 690
 691 static inline int
 692 qat_write_hw_desc_entry(struct rte_crypto_op *op, uint8_t *out_msg)
 693 {
 694         struct qat_session *ctx;
 695         struct icp_qat_fw_la_cipher_req_params *cipher_param;
 696         struct icp_qat_fw_la_auth_req_params *auth_param;
 697         register struct icp_qat_fw_la_bulk_req *qat_req;
 698
 699 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 700         if (unlikely(op->type != RTE_CRYPTO_OP_TYPE_SYMMETRIC)) {
 701                 PMD_DRV_LOG(ERR, "QAT PMD only supports symmetric crypto "
 702                                 "operation requests, op (%p) is not a "
 703                                 "symmetric operation.", op);
 704                 return -EINVAL;
 705         }
 706 #endif
 707         if (unlikely(op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS)) {
 708                 PMD_DRV_LOG(ERR, "QAT PMD only supports session oriented"
 709                                 " requests, op (%p) is sessionless.", op);
 710                 return -EINVAL;
 711         }
 712
 713         if (unlikely(op->sym->session->dev_type != RTE_CRYPTODEV_QAT_SYM_PMD)) {
 714                 PMD_DRV_LOG(ERR, "Session was not created for this device");
 715                 return -EINVAL;
 716         }
 717
 718         ctx = (struct qat_session *)op->sym->session->_private;
 719         qat_req = (struct icp_qat_fw_la_bulk_req *)out_msg;
 720         *qat_req = ctx->fw_req;
 721         qat_req->comn_mid.opaque_data = (uint64_t)(uintptr_t)op;
 722
 723         qat_req->comn_mid.dst_length =
 724                 qat_req->comn_mid.src_length =
 725                                 rte_pktmbuf_data_len(op->sym->m_src);
 726
 727         qat_req->comn_mid.dest_data_addr =
 728                 qat_req->comn_mid.src_data_addr =
 729                             rte_pktmbuf_mtophys(op->sym->m_src);
 730
 731         if (unlikely(op->sym->m_dst != NULL)) {
 732                 qat_req->comn_mid.dest_data_addr =
 733                                 rte_pktmbuf_mtophys(op->sym->m_dst);
 734                 qat_req->comn_mid.dst_length =
 735                                 rte_pktmbuf_data_len(op->sym->m_dst);
 736         }
 737
 738         cipher_param = (void *)&qat_req->serv_specif_rqpars;
 739         auth_param = (void *)((uint8_t *)cipher_param + sizeof(*cipher_param));
 740
 741         cipher_param->cipher_length = op->sym->cipher.data.length;
 742         cipher_param->cipher_offset = op->sym->cipher.data.offset;
 743         if (ctx->qat_cipher_alg == ICP_QAT_HW_CIPHER_ALGO_SNOW_3G_UEA2) {
 744                 if (unlikely((cipher_param->cipher_length % BYTE_LENGTH != 0) ||
 745                                 (cipher_param->cipher_offset
 746                                         % BYTE_LENGTH != 0))) {
 747                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 748                                 "supports byte aligned values");
 749                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 750                         return -EINVAL;
 751                 }
 752                 cipher_param->cipher_length >>= 3;
 753                 cipher_param->cipher_offset >>= 3;
 754         }
 755
 756         if (op->sym->cipher.iv.length && (op->sym->cipher.iv.length <=
 757                         sizeof(cipher_param->u.cipher_IV_array))) {
 758                 rte_memcpy(cipher_param->u.cipher_IV_array,
 759                                 op->sym->cipher.iv.data,
 760                                 op->sym->cipher.iv.length);
 761         } else {
 762                 ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(
 763                                 qat_req->comn_hdr.serv_specif_flags,
 764                                 ICP_QAT_FW_CIPH_IV_64BIT_PTR);
 765                 cipher_param->u.s.cipher_IV_ptr = op->sym->cipher.iv.phys_addr;
 766         }
 767         if (op->sym->auth.digest.phys_addr) {
 768                 ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(
 769                                 qat_req->comn_hdr.serv_specif_flags,
 770                                 ICP_QAT_FW_LA_NO_DIGEST_IN_BUFFER);
 771                 auth_param->auth_res_addr = op->sym->auth.digest.phys_addr;
 772         }
 773         auth_param->auth_off = op->sym->auth.data.offset;
 774         auth_param->auth_len = op->sym->auth.data.length;
 775         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_SNOW_3G_UIA2) {
 776                 if (unlikely((auth_param->auth_off % BYTE_LENGTH != 0) ||
 777                                 (auth_param->auth_len % BYTE_LENGTH != 0))) {
 778                         PMD_DRV_LOG(ERR, " For Snow3g, QAT PMD only "
 779                                 "supports byte aligned values");
 780                         op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 781                         return -EINVAL;
 782                 }
 783                 auth_param->auth_off >>= 3;
 784                 auth_param->auth_len >>= 3;
 785         }
 786         auth_param->u1.aad_adr = op->sym->auth.aad.phys_addr;
 787         /* (GCM) aad length(240 max) will be at this location after precompute */
 788         if (ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_128 ||
 789                         ctx->qat_hash_alg == ICP_QAT_HW_AUTH_ALGO_GALOIS_64) {
 790                 struct icp_qat_hw_auth_algo_blk *hash;
 791
 792                 if (ctx->qat_cmd == ICP_QAT_FW_LA_CMD_HASH_CIPHER)
 793                         hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&ctx->cd);
 794                 else
 795                         hash = (struct icp_qat_hw_auth_algo_blk *)((char *)&ctx->cd +
 796                                 sizeof(struct icp_qat_hw_cipher_algo_blk));
 797
 798                 auth_param->u2.aad_sz = ALIGN_POW2_ROUNDUP(hash->sha.state1[
 799                                         ICP_QAT_HW_GALOIS_128_STATE1_SZ +
 800                                         ICP_QAT_HW_GALOIS_H_SZ + 3], 16);
 801                 if (op->sym->cipher.iv.length == 12) {
 802                         /*
 803                          * For GCM a 12 bit IV is allowed,
 804                          * but we need to inform the f/w
 805                          */
 806                         ICP_QAT_FW_LA_GCM_IV_LEN_FLAG_SET(
 807                                 qat_req->comn_hdr.serv_specif_flags,
 808                                 ICP_QAT_FW_LA_GCM_IV_LEN_12_OCTETS);
 809                 }
 810         }
 811         auth_param->hash_state_sz = (auth_param->u2.aad_sz) >> 3;
 812
 813
 814 #ifdef RTE_LIBRTE_PMD_QAT_DEBUG_TX
 815         rte_hexdump(stdout, "qat_req:", qat_req,
 816                         sizeof(struct icp_qat_fw_la_bulk_req));
 817         rte_hexdump(stdout, "src_data:",
 818                         rte_pktmbuf_mtod(op->sym->m_src, uint8_t*),
 819                         rte_pktmbuf_data_len(op->sym->m_src));
 820         rte_hexdump(stdout, "iv:", op->sym->cipher.iv.data,
 821                         op->sym->cipher.iv.length);
 822         rte_hexdump(stdout, "digest:", op->sym->auth.digest.data,
 823                         op->sym->auth.digest.length);
 824         rte_hexdump(stdout, "aad:", op->sym->auth.aad.data,
 825                         op->sym->auth.aad.length);
 826 #endif
 827         return 0;
 828 }
 829
 830 static inline uint32_t adf_modulo(uint32_t data, uint32_t shift)
 831 {
 832         uint32_t div = data >> shift;
 833         uint32_t mult = div << shift;
 834
 835         return data - mult;
 836 }
 837
 838 void qat_crypto_sym_session_init(struct rte_mempool *mp, void *sym_sess)
 839 {
 840         struct rte_cryptodev_sym_session *sess = sym_sess;
 841         struct qat_session *s = (void *)sess->_private;
 842
 843         PMD_INIT_FUNC_TRACE();
 844         s->cd_paddr = rte_mempool_virt2phy(mp, sess) +
 845                 offsetof(struct qat_session, cd) +
 846                 offsetof(struct rte_cryptodev_sym_session, _private);
 847 }
 848
 849 int qat_dev_config(__rte_unused struct rte_cryptodev *dev)
 850 {
 851         PMD_INIT_FUNC_TRACE();
 852         return -ENOTSUP;
 853 }
 854
 855 int qat_dev_start(__rte_unused struct rte_cryptodev *dev)
 856 {
 857         PMD_INIT_FUNC_TRACE();
 858         return 0;
 859 }
 860
 861 void qat_dev_stop(__rte_unused struct rte_cryptodev *dev)
 862 {
 863         PMD_INIT_FUNC_TRACE();
 864 }
 865
 866 int qat_dev_close(struct rte_cryptodev *dev)
 867 {
 868         int i, ret;
 869
 870         PMD_INIT_FUNC_TRACE();
 871
 872         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 873                 ret = qat_crypto_sym_qp_release(dev, i);
 874                 if (ret < 0)
 875                         return ret;
 876         }
 877
 878         return 0;
 879 }
 880
 881 void qat_dev_info_get(__rte_unused struct rte_cryptodev *dev,
 882                                 struct rte_cryptodev_info *info)
 883 {
 884         struct qat_pmd_private *internals = dev->data->dev_private;
 885
 886         PMD_INIT_FUNC_TRACE();
 887         if (info != NULL) {
 888                 info->max_nb_queue_pairs =
 889                                 ADF_NUM_SYM_QPS_PER_BUNDLE *
 890                                 ADF_NUM_BUNDLES_PER_DEV;
 891                 info->feature_flags = dev->feature_flags;
 892                 info->capabilities = qat_pmd_capabilities;
 893                 info->sym.max_nb_sessions = internals->max_nb_sessions;
 894                 info->dev_type = RTE_CRYPTODEV_QAT_SYM_PMD;
 895         }
 896 }
 897
 898 void qat_crypto_sym_stats_get(struct rte_cryptodev *dev,
 899                 struct rte_cryptodev_stats *stats)
 900 {
 901         int i;
 902         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
 903
 904         PMD_INIT_FUNC_TRACE();
 905         if (stats == NULL) {
 906                 PMD_DRV_LOG(ERR, "invalid stats ptr NULL");
 907                 return;
 908         }
 909         for (i = 0; i < dev->data->nb_queue_pairs; i++) {
 910                 if (qp[i] == NULL) {
 911                         PMD_DRV_LOG(DEBUG, "Uninitialised queue pair");
 912                         continue;
 913                 }
 914
 915                 stats->enqueued_count += qp[i]->stats.enqueued_count;
 916                 stats->dequeued_count += qp[i]->stats.enqueued_count;
 917                 stats->enqueue_err_count += qp[i]->stats.enqueue_err_count;
 918                 stats->dequeue_err_count += qp[i]->stats.enqueue_err_count;
 919         }
 920 }
 921
 922 void qat_crypto_sym_stats_reset(struct rte_cryptodev *dev)
 923 {
 924         int i;
 925         struct qat_qp **qp = (struct qat_qp **)(dev->data->queue_pairs);
 926
 927         PMD_INIT_FUNC_TRACE();
 928         for (i = 0; i < dev->data->nb_queue_pairs; i++)
 929                 memset(&(qp[i]->stats), 0, sizeof(qp[i]->stats));
 930         PMD_DRV_LOG(DEBUG, "QAT crypto: stats cleared");
 931 }