examples/dpdk_qat/crypto.c

   1 /*-
   2  *   BSD LICENSE
   3  *
   4  *   Copyright(c) 2010-2014 Intel Corporation. All rights reserved.
   5  *   All rights reserved.
   6  *
   7  *   Redistribution and use in source and binary forms, with or without
   8  *   modification, are permitted provided that the following conditions
   9  *   are met:
  10  *
  11  *     * Redistributions of source code must retain the above copyright
  12  *       notice, this list of conditions and the following disclaimer.
  13  *     * Redistributions in binary form must reproduce the above copyright
  14  *       notice, this list of conditions and the following disclaimer in
  15  *       the documentation and/or other materials provided with the
  16  *       distribution.
  17  *     * Neither the name of Intel Corporation nor the names of its
  18  *       contributors may be used to endorse or promote products derived
  19  *       from this software without specific prior written permission.
  20  *
  21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  32  */
  33
  34 #include <stdio.h>
  35 #include <stdlib.h>
  36 #include <strings.h>
  37 #include <string.h>
  38 #include <inttypes.h>
  39 #include <errno.h>
  40 #include <sys/queue.h>
  41 #include <stdarg.h>
  42
  43 #include <rte_common.h>
  44 #include <rte_log.h>
  45 #include <rte_debug.h>
  46 #include <rte_memory.h>
  47 #include <rte_memzone.h>
  48 #include <rte_ether.h>
  49 #include <rte_malloc.h>
  50 #include <rte_launch.h>
  51 #include <rte_eal.h>
  52 #include <rte_per_lcore.h>
  53 #include <rte_lcore.h>
  54 #include <rte_atomic.h>
  55 #include <rte_branch_prediction.h>
  56 #include <rte_mempool.h>
  57 #include <rte_mbuf.h>
  58 #include <rte_string_fns.h>
  59
  60 #define CPA_CY_SYM_DP_TMP_WORKAROUND 1
  61
  62 #include "cpa.h"
  63 #include "cpa_types.h"
  64 #include "cpa_cy_sym_dp.h"
  65 #include "cpa_cy_common.h"
  66 #include "cpa_cy_im.h"
  67 #include "icp_sal_user.h"
  68 #include "icp_sal_poll.h"
  69
  70 #include "crypto.h"
  71
  72 /* CIPHER KEY LENGTHS */
  73 #define KEY_SIZE_64_IN_BYTES    (64 / 8)
  74 #define KEY_SIZE_56_IN_BYTES    (56 / 8)
  75 #define KEY_SIZE_128_IN_BYTES   (128 / 8)
  76 #define KEY_SIZE_168_IN_BYTES   (168 / 8)
  77 #define KEY_SIZE_192_IN_BYTES   (192 / 8)
  78 #define KEY_SIZE_256_IN_BYTES   (256 / 8)
  79
  80 /* HMAC AUTH KEY LENGTHS */
  81 #define AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES       (128 / 8)
  82 #define SHA1_AUTH_KEY_LENGTH_IN_BYTES           (160 / 8)
  83 #define SHA224_AUTH_KEY_LENGTH_IN_BYTES         (224 / 8)
  84 #define SHA256_AUTH_KEY_LENGTH_IN_BYTES         (256 / 8)
  85 #define SHA384_AUTH_KEY_LENGTH_IN_BYTES         (384 / 8)
  86 #define SHA512_AUTH_KEY_LENGTH_IN_BYTES         (512 / 8)
  87 #define MD5_AUTH_KEY_LENGTH_IN_BYTES            (128 / 8)
  88 #define KASUMI_AUTH_KEY_LENGTH_IN_BYTES         (128 / 8)
  89
  90 /* HASH DIGEST LENGHTS */
  91 #define AES_XCBC_DIGEST_LENGTH_IN_BYTES         (128 / 8)
  92 #define AES_XCBC_96_DIGEST_LENGTH_IN_BYTES      (96 / 8)
  93 #define MD5_DIGEST_LENGTH_IN_BYTES              (128 / 8)
  94 #define SHA1_DIGEST_LENGTH_IN_BYTES             (160 / 8)
  95 #define SHA1_96_DIGEST_LENGTH_IN_BYTES          (96 / 8)
  96 #define SHA224_DIGEST_LENGTH_IN_BYTES           (224 / 8)
  97 #define SHA256_DIGEST_LENGTH_IN_BYTES           (256 / 8)
  98 #define SHA384_DIGEST_LENGTH_IN_BYTES           (384 / 8)
  99 #define SHA512_DIGEST_LENGTH_IN_BYTES           (512 / 8)
 100 #define KASUMI_DIGEST_LENGTH_IN_BYTES           (32 / 8)
 101
 102 #define IV_LENGTH_16_BYTES      (16)
 103 #define IV_LENGTH_8_BYTES       (8)
 104
 105
 106 /*
 107  * rte_memzone is used to allocate physically contiguous virtual memory.
 108  * In this application we allocate a single block and divide between variables
 109  * which require a virtual to physical mapping for use by the QAT driver.
 110  * Virt2phys is only performed during initialisation and not on the data-path.
 111  */
 112
 113 #define LCORE_MEMZONE_SIZE      (1 << 22)
 114
 115 struct lcore_memzone
 116 {
 117         const struct rte_memzone *memzone;
 118         void *next_free_address;
 119 };
 120
 121 /*
 122  * Size the qa software response queue.
 123  * Note: Head and Tail are 8 bit, therefore, the queue is
 124  * fixed to 256 entries.
 125  */
 126 #define CRYPTO_SOFTWARE_QUEUE_SIZE 256
 127
 128 struct qa_callbackQueue {
 129         uint8_t head;
 130         uint8_t tail;
 131         uint16_t numEntries;
 132         struct rte_mbuf *qaCallbackRing[CRYPTO_SOFTWARE_QUEUE_SIZE];
 133 };
 134
 135 struct qa_core_conf {
 136         CpaCySymDpSessionCtx *encryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 137         CpaCySymDpSessionCtx *decryptSessionHandleTbl[NUM_CRYPTO][NUM_HMAC];
 138         CpaInstanceHandle instanceHandle;
 139         struct qa_callbackQueue callbackQueue;
 140         uint64_t qaOutstandingRequests;
 141         uint64_t numResponseAttempts;
 142         uint8_t kickFreq;
 143         void *pPacketIV;
 144         CpaPhysicalAddr packetIVPhy;
 145         struct lcore_memzone lcoreMemzone;
 146 } __rte_cache_aligned;
 147
 148 #define MAX_CORES   (RTE_MAX_LCORE)
 149
 150 static struct qa_core_conf qaCoreConf[MAX_CORES];
 151
 152 /*
 153  *Create maximum possible key size,
 154  *One for cipher and one for hash
 155  */
 156 struct glob_keys {
 157         uint8_t cipher_key[32];
 158         uint8_t hash_key[64];
 159         uint8_t iv[16];
 160 };
 161
 162 struct glob_keys g_crypto_hash_keys = {
 163         .cipher_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 164                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 165                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 166                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20},
 167         .hash_key = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 168                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10,
 169                 0x11,0x12,0x13,0x14,0x15,0x16,0x17,0x18,
 170                 0x19,0x1a,0x1b,0x1c,0x1d,0x1e,0x1f,0x20,
 171                 0x21,0x22,0x23,0x24,0x25,0x26,0x27,0x28,
 172                 0x29,0x2a,0x2b,0x2c,0x2d,0x2e,0x2f,0x30,
 173                 0x31,0x32,0x33,0x34,0x35,0x36,0x37,0x38,
 174                 0x39,0x4a,0x4b,0x4c,0x4d,0x4e,0x4f,0x50},
 175         .iv = {0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,
 176                 0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f,0x10}
 177 };
 178
 179 /*
 180  * Offsets from the start of the packet.
 181  *
 182  */
 183 #define PACKET_DATA_START_PHYS(p) \
 184                 ((p)->buf_physaddr + (p)->data_off)
 185
 186 /*
 187  * A fixed offset to where the crypto is to be performed, which is the first
 188  * byte after the Ethernet(14 bytes) and IPv4 headers(20 bytes)
 189  */
 190 #define CRYPTO_START_OFFSET             (14+20)
 191 #define HASH_START_OFFSET               (14+20)
 192 #define CIPHER_BLOCK_DEFAULT_SIZE       (16)
 193 #define HASH_BLOCK_DEFAULT_SIZE         (16)
 194
 195 /*
 196  * Offset to the opdata from the start of the data portion of packet.
 197  * Assumption: The buffer is physically contiguous.
 198  * +18 takes this to the next cache line.
 199  */
 200
 201 #define CRYPTO_OFFSET_TO_OPDATA         (ETHER_MAX_LEN+18)
 202
 203 /*
 204  * Default number of requests to place on the hardware ring before kicking the
 205  * ring pointers.
 206  */
 207 #define CRYPTO_BURST_TX (16)
 208
 209 /*
 210  * Only call the qa poll function when the number responses in the software
 211  * queue drops below this number.
 212  */
 213 #define CRYPTO_QUEUED_RESP_POLL_THRESHOLD       (32)
 214
 215 /*
 216  * Limit the number of polls per call to get_next_response.
 217  */
 218 #define GET_NEXT_RESPONSE_FREQ  (32)
 219
 220 /*
 221  * Max number of responses to pull from the qa in one poll.
 222  */
 223 #define CRYPTO_MAX_RESPONSE_QUOTA \
 224                 (CRYPTO_SOFTWARE_QUEUE_SIZE-CRYPTO_QUEUED_RESP_POLL_THRESHOLD-1)
 225
 226 #if (CRYPTO_QUEUED_RESP_POLL_THRESHOLD + CRYPTO_MAX_RESPONSE_QUOTA >= \
 227                 CRYPTO_SOFTWARE_QUEUE_SIZE)
 228 #error Its possible to overflow the qa response Q with current poll and \
 229                 response quota.
 230 #endif
 231
 232 static void
 233 crypto_callback(CpaCySymDpOpData *pOpData,
 234                 __rte_unused CpaStatus status,
 235                 __rte_unused CpaBoolean verifyResult)
 236 {
 237         uint32_t lcore_id;
 238         lcore_id = rte_lcore_id();
 239         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 240
 241         /*
 242          * Received a completion from the QA hardware.
 243          * Place the response on the return queue.
 244          */
 245         callbackQ->qaCallbackRing[callbackQ->head] = pOpData->pCallbackTag;
 246         callbackQ->head++;
 247         callbackQ->numEntries++;
 248         qaCoreConf[lcore_id].qaOutstandingRequests--;
 249 }
 250
 251 static void
 252 qa_crypto_callback(CpaCySymDpOpData *pOpData, CpaStatus status,
 253                 CpaBoolean verifyResult)
 254 {
 255         crypto_callback(pOpData, status, verifyResult);
 256 }
 257
 258 /*
 259  * Each allocation from a particular memzone lasts for the life-time of
 260  * the application. No freeing of previous allocations will occur.
 261  */
 262 static void *
 263 alloc_memzone_region(uint32_t length, uint32_t lcore_id)
 264 {
 265         char *current_free_addr_ptr = NULL;
 266         struct lcore_memzone *lcore_memzone = &(qaCoreConf[lcore_id].lcoreMemzone);
 267
 268         current_free_addr_ptr  = lcore_memzone->next_free_address;
 269
 270         if (current_free_addr_ptr + length >=
 271                 (char *)lcore_memzone->memzone->addr + lcore_memzone->memzone->len) {
 272                 printf("Crypto: No memory available in memzone\n");
 273                 return NULL;
 274         }
 275         lcore_memzone->next_free_address = current_free_addr_ptr + length;
 276
 277         return (void *)current_free_addr_ptr;
 278 }
 279
 280 /*
 281  * Virtual to Physical Address translation is only executed during initialization
 282  * and not on the data-path.
 283  */
 284 static CpaPhysicalAddr
 285 qa_v2p(void *ptr)
 286 {
 287         const struct rte_memzone *memzone = NULL;
 288         uint32_t lcore_id = 0;
 289         RTE_LCORE_FOREACH(lcore_id) {
 290                 memzone = qaCoreConf[lcore_id].lcoreMemzone.memzone;
 291
 292                 if ((char*) ptr >= (char *) memzone->addr &&
 293                                 (char*) ptr < ((char*) memzone->addr + memzone->len)) {
 294                         return (CpaPhysicalAddr)
 295                                         (memzone->phys_addr + ((char *) ptr - (char*) memzone->addr));
 296                 }
 297         }
 298         printf("Crypto: Corresponding physical address not found in memzone\n");
 299         return (CpaPhysicalAddr) 0;
 300 }
 301
 302 static CpaStatus
 303 getCoreAffinity(Cpa32U *coreAffinity, const CpaInstanceHandle instanceHandle)
 304 {
 305         CpaInstanceInfo2 info;
 306         Cpa16U i = 0;
 307         CpaStatus status = CPA_STATUS_SUCCESS;
 308
 309         memset(&info, 0, sizeof(CpaInstanceInfo2));
 310
 311         status = cpaCyInstanceGetInfo2(instanceHandle, &info);
 312         if (CPA_STATUS_SUCCESS != status) {
 313                 printf("Crypto: Error getting instance info\n");
 314                 return CPA_STATUS_FAIL;
 315         }
 316         for (i = 0; i < MAX_CORES; i++) {
 317                 if (CPA_BITMAP_BIT_TEST(info.coreAffinity, i)) {
 318                         *coreAffinity = i;
 319                         return CPA_STATUS_SUCCESS;
 320                 }
 321         }
 322         return CPA_STATUS_FAIL;
 323 }
 324
 325 static CpaStatus
 326 get_crypto_instance_on_core(CpaInstanceHandle *pInstanceHandle,
 327                 uint32_t lcore_id)
 328 {
 329         Cpa16U numInstances = 0, i = 0;
 330         CpaStatus status = CPA_STATUS_FAIL;
 331         CpaInstanceHandle *pLocalInstanceHandles = NULL;
 332         Cpa32U coreAffinity = 0;
 333
 334         status = cpaCyGetNumInstances(&numInstances);
 335         if (CPA_STATUS_SUCCESS != status || numInstances == 0) {
 336                 return CPA_STATUS_FAIL;
 337         }
 338
 339         pLocalInstanceHandles = rte_malloc("pLocalInstanceHandles",
 340                         sizeof(CpaInstanceHandle) * numInstances, RTE_CACHE_LINE_SIZE);
 341
 342         if (NULL == pLocalInstanceHandles) {
 343                 return CPA_STATUS_FAIL;
 344         }
 345         status = cpaCyGetInstances(numInstances, pLocalInstanceHandles);
 346         if (CPA_STATUS_SUCCESS != status) {
 347                 printf("Crypto: cpaCyGetInstances failed with status: %"PRId32"\n", status);
 348                 rte_free((void *) pLocalInstanceHandles);
 349                 return CPA_STATUS_FAIL;
 350         }
 351
 352         for (i = 0; i < numInstances; i++) {
 353                 status = getCoreAffinity(&coreAffinity, pLocalInstanceHandles[i]);
 354                 if (CPA_STATUS_SUCCESS != status) {
 355                         rte_free((void *) pLocalInstanceHandles);
 356                         return CPA_STATUS_FAIL;
 357                 }
 358                 if (coreAffinity == lcore_id) {
 359                         printf("Crypto: instance found on core %d\n", i);
 360                         *pInstanceHandle = pLocalInstanceHandles[i];
 361                         return CPA_STATUS_SUCCESS;
 362                 }
 363         }
 364         /* core affinity not found */
 365         rte_free((void *) pLocalInstanceHandles);
 366         return CPA_STATUS_FAIL;
 367 }
 368
 369 static CpaStatus
 370 initCySymSession(const int pkt_cipher_alg,
 371                 const int pkt_hash_alg, const CpaCySymHashMode hashMode,
 372                 const CpaCySymCipherDirection crypto_direction,
 373                 CpaCySymSessionCtx **ppSessionCtx,
 374                 const CpaInstanceHandle cyInstanceHandle,
 375                 const uint32_t lcore_id)
 376 {
 377         Cpa32U sessionCtxSizeInBytes = 0;
 378         CpaStatus status = CPA_STATUS_FAIL;
 379         CpaBoolean isCrypto = CPA_TRUE, isHmac = CPA_TRUE;
 380         CpaCySymSessionSetupData sessionSetupData;
 381
 382         memset(&sessionSetupData, 0, sizeof(CpaCySymSessionSetupData));
 383
 384         /* Assumption: key length is set to each algorithm's max length */
 385         switch (pkt_cipher_alg) {
 386         case NO_CIPHER:
 387                 isCrypto = CPA_FALSE;
 388                 break;
 389         case CIPHER_DES:
 390                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 391                                 CPA_CY_SYM_CIPHER_DES_ECB;
 392                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 393                                 KEY_SIZE_64_IN_BYTES;
 394                 break;
 395         case CIPHER_DES_CBC:
 396                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 397                                 CPA_CY_SYM_CIPHER_DES_CBC;
 398                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 399                                 KEY_SIZE_64_IN_BYTES;
 400                 break;
 401         case CIPHER_DES3:
 402                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 403                                 CPA_CY_SYM_CIPHER_3DES_ECB;
 404                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 405                                 KEY_SIZE_192_IN_BYTES;
 406                 break;
 407         case CIPHER_DES3_CBC:
 408                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 409                                 CPA_CY_SYM_CIPHER_3DES_CBC;
 410                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 411                                 KEY_SIZE_192_IN_BYTES;
 412                 break;
 413         case CIPHER_AES:
 414                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 415                                 CPA_CY_SYM_CIPHER_AES_ECB;
 416                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 417                                 KEY_SIZE_128_IN_BYTES;
 418                 break;
 419         case CIPHER_AES_CBC_128:
 420                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 421                                 CPA_CY_SYM_CIPHER_AES_CBC;
 422                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 423                                 KEY_SIZE_128_IN_BYTES;
 424                 break;
 425         case CIPHER_KASUMI_F8:
 426                 sessionSetupData.cipherSetupData.cipherAlgorithm =
 427                                 CPA_CY_SYM_CIPHER_KASUMI_F8;
 428                 sessionSetupData.cipherSetupData.cipherKeyLenInBytes =
 429                                 KEY_SIZE_128_IN_BYTES;
 430                 break;
 431         default:
 432                 printf("Crypto: Undefined Cipher specified\n");
 433                 break;
 434         }
 435         /* Set the cipher direction */
 436         if (isCrypto) {
 437                 sessionSetupData.cipherSetupData.cipherDirection = crypto_direction;
 438                 sessionSetupData.cipherSetupData.pCipherKey =
 439                                 g_crypto_hash_keys.cipher_key;
 440                 sessionSetupData.symOperation = CPA_CY_SYM_OP_CIPHER;
 441         }
 442
 443         /* Setup Hash common fields */
 444         switch (pkt_hash_alg) {
 445         case NO_HASH:
 446                 isHmac = CPA_FALSE;
 447                 break;
 448         case HASH_AES_XCBC:
 449                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 450                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 451                                 AES_XCBC_DIGEST_LENGTH_IN_BYTES;
 452                 break;
 453         case HASH_AES_XCBC_96:
 454                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_AES_XCBC;
 455                                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 456                                 AES_XCBC_96_DIGEST_LENGTH_IN_BYTES;
 457                 break;
 458         case HASH_MD5:
 459                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_MD5;
 460                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 461                                 MD5_DIGEST_LENGTH_IN_BYTES;
 462                 break;
 463         case HASH_SHA1:
 464                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 465                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 466                                 SHA1_DIGEST_LENGTH_IN_BYTES;
 467                 break;
 468         case HASH_SHA1_96:
 469                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA1;
 470                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 471                                 SHA1_96_DIGEST_LENGTH_IN_BYTES;
 472             break;
 473         case HASH_SHA224:
 474                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA224;
 475                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 476                                 SHA224_DIGEST_LENGTH_IN_BYTES;
 477                 break;
 478         case HASH_SHA256:
 479                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA256;
 480                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 481                                 SHA256_DIGEST_LENGTH_IN_BYTES;
 482                 break;
 483         case HASH_SHA384:
 484                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA384;
 485                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 486                                 SHA384_DIGEST_LENGTH_IN_BYTES;
 487                 break;
 488         case HASH_SHA512:
 489                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_SHA512;
 490                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 491                                 SHA512_DIGEST_LENGTH_IN_BYTES;
 492                 break;
 493         case HASH_KASUMI_F9:
 494                 sessionSetupData.hashSetupData.hashAlgorithm = CPA_CY_SYM_HASH_KASUMI_F9;
 495                 sessionSetupData.hashSetupData.digestResultLenInBytes =
 496                                 KASUMI_DIGEST_LENGTH_IN_BYTES;
 497                 break;
 498         default:
 499                 printf("Crypto: Undefined Hash specified\n");
 500                 break;
 501         }
 502         if (isHmac) {
 503                 sessionSetupData.hashSetupData.hashMode = hashMode;
 504                 sessionSetupData.symOperation = CPA_CY_SYM_OP_HASH;
 505                 /* If using authenticated hash setup key lengths */
 506                 if (CPA_CY_SYM_HASH_MODE_AUTH == hashMode) {
 507                         /* Use a common max length key */
 508                         sessionSetupData.hashSetupData.authModeSetupData.authKey =
 509                                         g_crypto_hash_keys.hash_key;
 510                         switch (pkt_hash_alg) {
 511                         case HASH_AES_XCBC:
 512                         case HASH_AES_XCBC_96:
 513                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 514                                                 AES_XCBC_AUTH_KEY_LENGTH_IN_BYTES;
 515                                 break;
 516                         case HASH_MD5:
 517                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 518                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 519                                 break;
 520                         case HASH_SHA1:
 521                         case HASH_SHA1_96:
 522                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 523                                                 SHA1_AUTH_KEY_LENGTH_IN_BYTES;
 524                                 break;
 525                         case HASH_SHA224:
 526                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 527                                                 SHA224_AUTH_KEY_LENGTH_IN_BYTES;
 528                                 break;
 529                         case HASH_SHA256:
 530                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 531                                                 SHA256_AUTH_KEY_LENGTH_IN_BYTES;
 532                                 break;
 533                         case HASH_SHA384:
 534                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 535                                                 SHA384_AUTH_KEY_LENGTH_IN_BYTES;
 536                                 break;
 537                         case HASH_SHA512:
 538                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 539                                                 SHA512_AUTH_KEY_LENGTH_IN_BYTES;
 540                                 break;
 541                         case HASH_KASUMI_F9:
 542                                 sessionSetupData.hashSetupData.authModeSetupData.authKeyLenInBytes =
 543                                                 KASUMI_AUTH_KEY_LENGTH_IN_BYTES;
 544                                 break;
 545                         default:
 546                                 printf("Crypto: Undefined Hash specified\n");
 547                                 return CPA_STATUS_FAIL;
 548                         }
 549                 }
 550         }
 551
 552         /* Only high priority supported */
 553         sessionSetupData.sessionPriority = CPA_CY_PRIORITY_HIGH;
 554
 555         /* If chaining algorithms */
 556         if (isCrypto && isHmac) {
 557                 sessionSetupData.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
 558                 /* @assumption Alg Chain order is cipher then hash for encrypt
 559                  * and hash then cipher then has for decrypt*/
 560                 if (CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT == crypto_direction) {
 561                         sessionSetupData.algChainOrder =
 562                                         CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
 563                 } else {
 564                         sessionSetupData.algChainOrder =
 565                                         CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
 566                 }
 567         }
 568         if (!isCrypto && !isHmac) {
 569                 *ppSessionCtx = NULL;
 570                 return CPA_STATUS_SUCCESS;
 571         }
 572
 573         /* Set flags for digest operations */
 574         sessionSetupData.digestIsAppended = CPA_FALSE;
 575         sessionSetupData.verifyDigest = CPA_TRUE;
 576
 577         /* Get the session context size based on the crypto and/or hash operations*/
 578         status = cpaCySymDpSessionCtxGetSize(cyInstanceHandle, &sessionSetupData,
 579                         &sessionCtxSizeInBytes);
 580         if (CPA_STATUS_SUCCESS != status) {
 581                 printf("Crypto: cpaCySymDpSessionCtxGetSize error, status: %"PRId32"\n",
 582                                 status);
 583                 return CPA_STATUS_FAIL;
 584         }
 585
 586         *ppSessionCtx = alloc_memzone_region(sessionCtxSizeInBytes, lcore_id);
 587         if (NULL == *ppSessionCtx) {
 588                 printf("Crypto: Failed to allocate memory for Session Context\n");
 589                 return CPA_STATUS_FAIL;
 590         }
 591
 592         status = cpaCySymDpInitSession(cyInstanceHandle, &sessionSetupData,
 593                         *ppSessionCtx);
 594         if (CPA_STATUS_SUCCESS != status) {
 595                 printf("Crypto: cpaCySymDpInitSession failed with status %"PRId32"\n", status);
 596                 return CPA_STATUS_FAIL;
 597         }
 598         return CPA_STATUS_SUCCESS;
 599 }
 600
 601 static CpaStatus
 602 initSessionDataTables(struct qa_core_conf *qaCoreConf,uint32_t lcore_id)
 603 {
 604         Cpa32U i = 0, j = 0;
 605         CpaStatus status = CPA_STATUS_FAIL;
 606         for (i = 0; i < NUM_CRYPTO; i++) {
 607                 for (j = 0; j < NUM_HMAC; j++) {
 608                         if (((i == CIPHER_KASUMI_F8) && (j != NO_HASH) && (j != HASH_KASUMI_F9)) ||
 609                                 ((i != NO_CIPHER) && (i != CIPHER_KASUMI_F8) && (j == HASH_KASUMI_F9)))
 610                                 continue;
 611                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 612                                         CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT,
 613                                         &qaCoreConf->encryptSessionHandleTbl[i][j],
 614                                         qaCoreConf->instanceHandle,
 615                                         lcore_id);
 616                         if (CPA_STATUS_SUCCESS != status) {
 617                                 printf("Crypto: Failed to initialize Encrypt sessions\n");
 618                                 return CPA_STATUS_FAIL;
 619                         }
 620                         status = initCySymSession(i, j, CPA_CY_SYM_HASH_MODE_AUTH,
 621                                         CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT,
 622                                         &qaCoreConf->decryptSessionHandleTbl[i][j],
 623                                         qaCoreConf->instanceHandle,
 624                                         lcore_id);
 625                         if (CPA_STATUS_SUCCESS != status) {
 626                                 printf("Crypto: Failed to initialize Decrypt sessions\n");
 627                                 return CPA_STATUS_FAIL;
 628                         }
 629                 }
 630         }
 631         return CPA_STATUS_SUCCESS;
 632 }
 633
 634 int
 635 crypto_init(void)
 636 {
 637         if (CPA_STATUS_SUCCESS != icp_sal_userStartMultiProcess("SSL",CPA_FALSE)) {
 638                 printf("Crypto: Could not start sal for user space\n");
 639                 return CPA_STATUS_FAIL;
 640         }
 641         printf("Crypto: icp_sal_userStartMultiProcess(\"SSL\",CPA_FALSE)\n");
 642         return 0;
 643 }
 644
 645 /*
 646  * Per core initialisation
 647  */
 648 int
 649 per_core_crypto_init(uint32_t lcore_id)
 650 {
 651         CpaStatus status = CPA_STATUS_FAIL;
 652         char memzone_name[RTE_MEMZONE_NAMESIZE];
 653
 654         int socketID = rte_lcore_to_socket_id(lcore_id);
 655
 656         /* Allocate software ring for response messages. */
 657
 658         qaCoreConf[lcore_id].callbackQueue.head = 0;
 659         qaCoreConf[lcore_id].callbackQueue.tail = 0;
 660         qaCoreConf[lcore_id].callbackQueue.numEntries = 0;
 661         qaCoreConf[lcore_id].kickFreq = 0;
 662         qaCoreConf[lcore_id].qaOutstandingRequests = 0;
 663         qaCoreConf[lcore_id].numResponseAttempts = 0;
 664
 665         /* Initialise and reserve lcore memzone for virt2phys translation */
 666         snprintf(memzone_name,
 667                         RTE_MEMZONE_NAMESIZE,
 668                         "lcore_%u",
 669                         lcore_id);
 670
 671         qaCoreConf[lcore_id].lcoreMemzone.memzone = rte_memzone_reserve(
 672                         memzone_name,
 673                         LCORE_MEMZONE_SIZE,
 674                         socketID,
 675                         0);
 676         if (NULL == qaCoreConf[lcore_id].lcoreMemzone.memzone) {
 677                 printf("Crypto: Error allocating memzone on lcore %u\n",lcore_id);
 678                 return -1;
 679         }
 680         qaCoreConf[lcore_id].lcoreMemzone.next_free_address =
 681                                                         qaCoreConf[lcore_id].lcoreMemzone.memzone->addr;
 682
 683         qaCoreConf[lcore_id].pPacketIV = alloc_memzone_region(IV_LENGTH_16_BYTES,
 684                                                         lcore_id);
 685
 686         if (NULL == qaCoreConf[lcore_id].pPacketIV ) {
 687                 printf("Crypto: Failed to allocate memory for Initialization Vector\n");
 688                 return -1;
 689         }
 690
 691         memcpy(qaCoreConf[lcore_id].pPacketIV, &g_crypto_hash_keys.iv,
 692                         IV_LENGTH_16_BYTES);
 693
 694         qaCoreConf[lcore_id].packetIVPhy = qa_v2p(qaCoreConf[lcore_id].pPacketIV);
 695         if (0 == qaCoreConf[lcore_id].packetIVPhy) {
 696                 printf("Crypto: Invalid physical address for Initialization Vector\n");
 697                 return -1;
 698         }
 699
 700         /*
 701          * Obtain the instance handle that is mapped to the current lcore.
 702          * This can fail if an instance is not mapped to a bank which has been
 703          * affinitized to the current lcore.
 704          */
 705         status = get_crypto_instance_on_core(&(qaCoreConf[lcore_id].instanceHandle),
 706                         lcore_id);
 707         if (CPA_STATUS_SUCCESS != status) {
 708                 printf("Crypto: get_crypto_instance_on_core failed with status: %"PRId32"\n",
 709                                 status);
 710                 return -1;
 711         }
 712
 713         status = cpaCySymDpRegCbFunc(qaCoreConf[lcore_id].instanceHandle,
 714                         (CpaCySymDpCbFunc) qa_crypto_callback);
 715         if (CPA_STATUS_SUCCESS != status) {
 716                 printf("Crypto: cpaCySymDpRegCbFunc failed with status: %"PRId32"\n", status);
 717                 return -1;
 718         }
 719
 720         /*
 721          * Set the address translation callback for virtual to physcial address
 722          * mapping. This will be called by the QAT driver during initialisation only.
 723          */
 724         status = cpaCySetAddressTranslation(qaCoreConf[lcore_id].instanceHandle,
 725                         (CpaVirtualToPhysical) qa_v2p);
 726         if (CPA_STATUS_SUCCESS != status) {
 727                 printf("Crypto: cpaCySetAddressTranslation failed with status: %"PRId32"\n",
 728                                 status);
 729                 return -1;
 730         }
 731
 732         status = initSessionDataTables(&qaCoreConf[lcore_id],lcore_id);
 733         if (CPA_STATUS_SUCCESS != status) {
 734                 printf("Crypto: Failed to allocate all session tables.");
 735                 return -1;
 736         }
 737         return 0;
 738 }
 739
 740 static CpaStatus
 741 enqueueOp(CpaCySymDpOpData *opData, uint32_t lcore_id)
 742 {
 743
 744         CpaStatus status;
 745
 746         /*
 747          * Assumption is there is no requirement to do load balancing between
 748          * acceleration units - that is one acceleration unit is tied to a core.
 749          */
 750         opData->instanceHandle = qaCoreConf[lcore_id].instanceHandle;
 751
 752         if ((++qaCoreConf[lcore_id].kickFreq) % CRYPTO_BURST_TX == 0) {
 753                 status = cpaCySymDpEnqueueOp(opData, CPA_TRUE);
 754         } else {
 755                 status = cpaCySymDpEnqueueOp(opData, CPA_FALSE);
 756         }
 757
 758         qaCoreConf[lcore_id].qaOutstandingRequests++;
 759
 760         return status;
 761 }
 762
 763 void
 764 crypto_flush_tx_queue(uint32_t lcore_id)
 765 {
 766
 767         cpaCySymDpPerformOpNow(qaCoreConf[lcore_id].instanceHandle);
 768 }
 769
 770 enum crypto_result
 771 crypto_encrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 772 {
 773         CpaCySymDpOpData *opData =
 774                         rte_pktmbuf_mtod_offset(rte_buff, CpaCySymDpOpData *,
 775                                                 CRYPTO_OFFSET_TO_OPDATA);
 776         uint32_t lcore_id;
 777
 778         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 779                 return CRYPTO_RESULT_FAIL;
 780
 781         lcore_id = rte_lcore_id();
 782
 783         memset(opData, 0, sizeof(CpaCySymDpOpData));
 784
 785         opData->srcBuffer = opData->dstBuffer = PACKET_DATA_START_PHYS(rte_buff);
 786         opData->srcBufferLen = opData->dstBufferLen = rte_buff->data_len;
 787         opData->sessionCtx = qaCoreConf[lcore_id].encryptSessionHandleTbl[c][h];
 788         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 789                         + CRYPTO_OFFSET_TO_OPDATA;
 790         opData->pCallbackTag = rte_buff;
 791
 792         /* if no crypto or hash operations are specified return fail */
 793         if (NO_CIPHER == c && NO_HASH == h)
 794                 return CRYPTO_RESULT_FAIL;
 795
 796         if (NO_CIPHER != c) {
 797                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 798                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 799
 800                 if (CIPHER_AES_CBC_128 == c)
 801                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 802                 else
 803                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 804
 805                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 806                 opData->messageLenToCipherInBytes = rte_buff->data_len
 807                                 - CRYPTO_START_OFFSET;
 808                 /*
 809                  * Work around for padding, message length has to be a multiple of
 810                  * block size.
 811                  */
 812                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 813                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 814         }
 815
 816         if (NO_HASH != h) {
 817
 818                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 819                 opData->messageLenToHashInBytes = rte_buff->data_len
 820                                 - HASH_START_OFFSET;
 821                 /*
 822                  * Work around for padding, message length has to be a multiple of block
 823                  * size.
 824                  */
 825                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 826                                 % HASH_BLOCK_DEFAULT_SIZE;
 827
 828                 /*
 829                  * Assumption: Ok ignore the passed digest pointer and place HMAC at end
 830                  * of packet.
 831                  */
 832                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->data_len;
 833         }
 834
 835         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 836                 /*
 837                  * Failed to place a packet on the hardware queue.
 838                  * Most likely because the QA hardware is busy.
 839                  */
 840                 return CRYPTO_RESULT_FAIL;
 841         }
 842         return CRYPTO_RESULT_IN_PROGRESS;
 843 }
 844
 845 enum crypto_result
 846 crypto_decrypt(struct rte_mbuf *rte_buff, enum cipher_alg c, enum hash_alg h)
 847 {
 848
 849         CpaCySymDpOpData *opData = rte_pktmbuf_mtod_offset(rte_buff, void *,
 850                                                            CRYPTO_OFFSET_TO_OPDATA);
 851         uint32_t lcore_id;
 852
 853         if (unlikely(c >= NUM_CRYPTO || h >= NUM_HMAC))
 854                 return CRYPTO_RESULT_FAIL;
 855
 856         lcore_id = rte_lcore_id();
 857
 858         memset(opData, 0, sizeof(CpaCySymDpOpData));
 859
 860         opData->dstBuffer = opData->srcBuffer = PACKET_DATA_START_PHYS(rte_buff);
 861         opData->dstBufferLen = opData->srcBufferLen = rte_buff->data_len;
 862         opData->thisPhys = PACKET_DATA_START_PHYS(rte_buff)
 863                         + CRYPTO_OFFSET_TO_OPDATA;
 864         opData->sessionCtx = qaCoreConf[lcore_id].decryptSessionHandleTbl[c][h];
 865         opData->pCallbackTag = rte_buff;
 866
 867         /* if no crypto or hmac operations are specified return fail */
 868         if (NO_CIPHER == c && NO_HASH == h)
 869                 return CRYPTO_RESULT_FAIL;
 870
 871         if (NO_CIPHER != c) {
 872                 opData->pIv = qaCoreConf[lcore_id].pPacketIV;
 873                 opData->iv = qaCoreConf[lcore_id].packetIVPhy;
 874
 875                 if (CIPHER_AES_CBC_128 == c)
 876                         opData->ivLenInBytes = IV_LENGTH_16_BYTES;
 877                 else
 878                         opData->ivLenInBytes = IV_LENGTH_8_BYTES;
 879
 880                 opData->cryptoStartSrcOffsetInBytes = CRYPTO_START_OFFSET;
 881                 opData->messageLenToCipherInBytes = rte_buff->data_len
 882                                 - CRYPTO_START_OFFSET;
 883
 884                 /*
 885                  * Work around for padding, message length has to be a multiple of block
 886                  * size.
 887                  */
 888                 opData->messageLenToCipherInBytes -= opData->messageLenToCipherInBytes
 889                                 % CIPHER_BLOCK_DEFAULT_SIZE;
 890         }
 891         if (NO_HASH != h) {
 892                 opData->hashStartSrcOffsetInBytes = HASH_START_OFFSET;
 893                 opData->messageLenToHashInBytes = rte_buff->data_len
 894                                 - HASH_START_OFFSET;
 895                 /*
 896                  * Work around for padding, message length has to be a multiple of block
 897                  * size.
 898                  */
 899                 opData->messageLenToHashInBytes -= opData->messageLenToHashInBytes
 900                                 % HASH_BLOCK_DEFAULT_SIZE;
 901                 opData->digestResult = rte_buff->buf_physaddr + rte_buff->data_len;
 902         }
 903
 904         if (CPA_STATUS_SUCCESS != enqueueOp(opData, lcore_id)) {
 905                 /*
 906                  * Failed to place a packet on the hardware queue.
 907                  * Most likely because the QA hardware is busy.
 908                  */
 909                 return CRYPTO_RESULT_FAIL;
 910         }
 911         return CRYPTO_RESULT_IN_PROGRESS;
 912 }
 913
 914 void *
 915 crypto_get_next_response(void)
 916 {
 917         uint32_t lcore_id;
 918         lcore_id = rte_lcore_id();
 919         struct qa_callbackQueue *callbackQ = &(qaCoreConf[lcore_id].callbackQueue);
 920         void *entry = NULL;
 921
 922         if (callbackQ->numEntries) {
 923                 entry = callbackQ->qaCallbackRing[callbackQ->tail];
 924                 callbackQ->tail++;
 925                 callbackQ->numEntries--;
 926         }
 927
 928         /* If there are no outstanding requests no need to poll, return entry */
 929         if (qaCoreConf[lcore_id].qaOutstandingRequests == 0)
 930                 return entry;
 931
 932         if (callbackQ->numEntries < CRYPTO_QUEUED_RESP_POLL_THRESHOLD
 933                         && qaCoreConf[lcore_id].numResponseAttempts++
 934                                         % GET_NEXT_RESPONSE_FREQ == 0) {
 935                 /*
 936                  * Only poll the hardware when there is less than
 937                  * CRYPTO_QUEUED_RESP_POLL_THRESHOLD elements in the software queue
 938                  */
 939                 icp_sal_CyPollDpInstance(qaCoreConf[lcore_id].instanceHandle,
 940                                 CRYPTO_MAX_RESPONSE_QUOTA);
 941         }
 942         return entry;
 943 }