drivers/crypto/armv8/rte_armv8_pmd.c

   1 /*
   2  *   BSD LICENSE
   3  *
   4  *   Copyright (C) Cavium networks Ltd. 2017.
   5  *
   6  *   Redistribution and use in source and binary forms, with or without
   7  *   modification, are permitted provided that the following conditions
   8  *   are met:
   9  *
  10  *     * Redistributions of source code must retain the above copyright
  11  *       notice, this list of conditions and the following disclaimer.
  12  *     * Redistributions in binary form must reproduce the above copyright
  13  *       notice, this list of conditions and the following disclaimer in
  14  *       the documentation and/or other materials provided with the
  15  *       distribution.
  16  *     * Neither the name of Cavium networks nor the names of its
  17  *       contributors may be used to endorse or promote products derived
  18  *       from this software without specific prior written permission.
  19  *
  20  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  21  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  22  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  23  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  24  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  25  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  26  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  27  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  28  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  30  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #include <stdbool.h>
  34
  35 #include <rte_common.h>
  36 #include <rte_hexdump.h>
  37 #include <rte_cryptodev.h>
  38 #include <rte_cryptodev_pmd.h>
  39 #include <rte_vdev.h>
  40 #include <rte_malloc.h>
  41 #include <rte_cpuflags.h>
  42
  43 #include "armv8_crypto_defs.h"
  44
  45 #include "rte_armv8_pmd_private.h"
  46
  47 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev);
  48
  49 /**
  50  * Pointers to the supported combined mode crypto functions are stored
  51  * in the static tables. Each combined (chained) cryptographic operation
  52  * can be described by a set of numbers:
  53  * - order:     order of operations (cipher, auth) or (auth, cipher)
  54  * - direction: encryption or decryption
  55  * - calg:      cipher algorithm such as AES_CBC, AES_CTR, etc.
  56  * - aalg:      authentication algorithm such as SHA1, SHA256, etc.
  57  * - keyl:      cipher key length, for example 128, 192, 256 bits
  58  *
  59  * In order to quickly acquire each function pointer based on those numbers,
  60  * a hierarchy of arrays is maintained. The final level, 3D array is indexed
  61  * by the combined mode function parameters only (cipher algorithm,
  62  * authentication algorithm and key length).
  63  *
  64  * This gives 3 memory accesses to obtain a function pointer instead of
  65  * traversing the array manually and comparing function parameters on each loop.
  66  *
  67  *                   +--+CRYPTO_FUNC
  68  *            +--+ENC|
  69  *      +--+CA|
  70  *      |     +--+DEC
  71  * ORDER|
  72  *      |     +--+ENC
  73  *      +--+AC|
  74  *            +--+DEC
  75  *
  76  */
  77
  78 /**
  79  * 3D array type for ARM Combined Mode crypto functions pointers.
  80  * CRYPTO_CIPHER_MAX:                   max cipher ID number
  81  * CRYPTO_AUTH_MAX:                     max auth ID number
  82  * CRYPTO_CIPHER_KEYLEN_MAX:            max key length ID number
  83  */
  84 typedef const crypto_func_t
  85 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
  86
  87 /* Evaluate to key length definition */
  88 #define KEYL(keyl)              (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl)
  89
  90 /* Local aliases for supported ciphers */
  91 #define CIPH_AES_CBC            RTE_CRYPTO_CIPHER_AES_CBC
  92 /* Local aliases for supported hashes */
  93 #define AUTH_SHA1_HMAC          RTE_CRYPTO_AUTH_SHA1_HMAC
  94 #define AUTH_SHA256_HMAC        RTE_CRYPTO_AUTH_SHA256_HMAC
  95
  96 /**
  97  * Arrays containing pointers to particular cryptographic,
  98  * combined mode functions.
  99  * crypto_op_ca_encrypt:        cipher (encrypt), authenticate
 100  * crypto_op_ca_decrypt:        cipher (decrypt), authenticate
 101  * crypto_op_ac_encrypt:        authenticate, cipher (encrypt)
 102  * crypto_op_ac_decrypt:        authenticate, cipher (decrypt)
 103  */
 104 static const crypto_func_tbl_t
 105 crypto_op_ca_encrypt = {
 106         /* [cipher alg][auth alg][key length] = crypto_function, */
 107         [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac,
 108         [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac,
 109 };
 110
 111 static const crypto_func_tbl_t
 112 crypto_op_ca_decrypt = {
 113         NULL
 114 };
 115
 116 static const crypto_func_tbl_t
 117 crypto_op_ac_encrypt = {
 118         NULL
 119 };
 120
 121 static const crypto_func_tbl_t
 122 crypto_op_ac_decrypt = {
 123         /* [cipher alg][auth alg][key length] = crypto_function, */
 124         [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec,
 125         [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec,
 126 };
 127
 128 /**
 129  * Arrays containing pointers to particular cryptographic function sets,
 130  * covering given cipher operation directions (encrypt, decrypt)
 131  * for each order of cipher and authentication pairs.
 132  */
 133 static const crypto_func_tbl_t *
 134 crypto_cipher_auth[] = {
 135         &crypto_op_ca_encrypt,
 136         &crypto_op_ca_decrypt,
 137         NULL
 138 };
 139
 140 static const crypto_func_tbl_t *
 141 crypto_auth_cipher[] = {
 142         &crypto_op_ac_encrypt,
 143         &crypto_op_ac_decrypt,
 144         NULL
 145 };
 146
 147 /**
 148  * Top level array containing pointers to particular cryptographic
 149  * function sets, covering given order of chained operations.
 150  * crypto_cipher_auth:  cipher first, authenticate after
 151  * crypto_auth_cipher:  authenticate first, cipher after
 152  */
 153 static const crypto_func_tbl_t **
 154 crypto_chain_order[] = {
 155         crypto_cipher_auth,
 156         crypto_auth_cipher,
 157         NULL
 158 };
 159
 160 /**
 161  * Extract particular combined mode crypto function from the 3D array.
 162  */
 163 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl)                   \
 164 ({                                                                      \
 165         crypto_func_tbl_t *func_tbl =                                   \
 166                                 (crypto_chain_order[(order)])[(cop)];   \
 167                                                                         \
 168         ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]);              \
 169 })
 170
 171 /*----------------------------------------------------------------------------*/
 172
 173 /**
 174  * 2D array type for ARM key schedule functions pointers.
 175  * CRYPTO_CIPHER_MAX:                   max cipher ID number
 176  * CRYPTO_CIPHER_KEYLEN_MAX:            max key length ID number
 177  */
 178 typedef const crypto_key_sched_t
 179 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
 180
 181 static const crypto_key_sched_tbl_t
 182 crypto_key_sched_encrypt = {
 183         /* [cipher alg][key length] = key_expand_func, */
 184         [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc,
 185 };
 186
 187 static const crypto_key_sched_tbl_t
 188 crypto_key_sched_decrypt = {
 189         /* [cipher alg][key length] = key_expand_func, */
 190         [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec,
 191 };
 192
 193 /**
 194  * Top level array containing pointers to particular key generation
 195  * function sets, covering given operation direction.
 196  * crypto_key_sched_encrypt:    keys for encryption
 197  * crypto_key_sched_decrypt:    keys for decryption
 198  */
 199 static const crypto_key_sched_tbl_t *
 200 crypto_key_sched_dir[] = {
 201         &crypto_key_sched_encrypt,
 202         &crypto_key_sched_decrypt,
 203         NULL
 204 };
 205
 206 /**
 207  * Extract particular combined mode crypto function from the 3D array.
 208  */
 209 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl)                           \
 210 ({                                                                      \
 211         crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)];   \
 212                                                                         \
 213         ((*ks_tbl)[(calg)][KEYL(keyl)]);                                \
 214 })
 215
 216 /*----------------------------------------------------------------------------*/
 217
 218 /*
 219  *------------------------------------------------------------------------------
 220  * Session Prepare
 221  *------------------------------------------------------------------------------
 222  */
 223
 224 /** Get xform chain order */
 225 static enum armv8_crypto_chain_order
 226 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
 227 {
 228
 229         /*
 230          * This driver currently covers only chained operations.
 231          * Ignore only cipher or only authentication operations
 232          * or chains longer than 2 xform structures.
 233          */
 234         if (xform->next == NULL || xform->next->next != NULL)
 235                 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
 236
 237         if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
 238                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
 239                         return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER;
 240         }
 241
 242         if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
 243                 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
 244                         return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH;
 245         }
 246
 247         return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
 248 }
 249
 250 static inline void
 251 auth_hmac_pad_prepare(struct armv8_crypto_session *sess,
 252                                 const struct rte_crypto_sym_xform *xform)
 253 {
 254         size_t i;
 255
 256         /* Generate i_key_pad and o_key_pad */
 257         memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad));
 258         rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key,
 259                                                         xform->auth.key.length);
 260         memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad));
 261         rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key,
 262                                                         xform->auth.key.length);
 263         /*
 264          * XOR key with IPAD/OPAD values to obtain i_key_pad
 265          * and o_key_pad.
 266          * Byte-by-byte operation may seem to be the less efficient
 267          * here but in fact it's the opposite.
 268          * The result ASM code is likely operate on NEON registers
 269          * (load auth key to Qx, load IPAD/OPAD to multiple
 270          * elements of Qy, eor 128 bits at once).
 271          */
 272         for (i = 0; i < SHA_BLOCK_MAX; i++) {
 273                 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE;
 274                 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE;
 275         }
 276 }
 277
 278 static inline int
 279 auth_set_prerequisites(struct armv8_crypto_session *sess,
 280                         const struct rte_crypto_sym_xform *xform)
 281 {
 282         uint8_t partial[64] = { 0 };
 283         int error;
 284
 285         switch (xform->auth.algo) {
 286         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 287                 /*
 288                  * Generate authentication key, i_key_pad and o_key_pad.
 289                  */
 290                 /* Zero memory under key */
 291                 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE);
 292
 293                 /*
 294                  * Now copy the given authentication key to the session
 295                  * key.
 296                  */
 297                 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
 298                                                 xform->auth.key.length);
 299
 300                 /* Prepare HMAC padding: key|pattern */
 301                 auth_hmac_pad_prepare(sess, xform);
 302                 /*
 303                  * Calculate partial hash values for i_key_pad and o_key_pad.
 304                  * Will be used as initialization state for final HMAC.
 305                  */
 306                 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad,
 307                     partial, SHA1_BLOCK_SIZE);
 308                 if (error != 0)
 309                         return -1;
 310                 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE);
 311
 312                 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad,
 313                     partial, SHA1_BLOCK_SIZE);
 314                 if (error != 0)
 315                         return -1;
 316                 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE);
 317
 318                 break;
 319         case RTE_CRYPTO_AUTH_SHA256_HMAC:
 320                 /*
 321                  * Generate authentication key, i_key_pad and o_key_pad.
 322                  */
 323                 /* Zero memory under key */
 324                 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE);
 325
 326                 /*
 327                  * Now copy the given authentication key to the session
 328                  * key.
 329                  */
 330                 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
 331                                                 xform->auth.key.length);
 332
 333                 /* Prepare HMAC padding: key|pattern */
 334                 auth_hmac_pad_prepare(sess, xform);
 335                 /*
 336                  * Calculate partial hash values for i_key_pad and o_key_pad.
 337                  * Will be used as initialization state for final HMAC.
 338                  */
 339                 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad,
 340                     partial, SHA256_BLOCK_SIZE);
 341                 if (error != 0)
 342                         return -1;
 343                 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE);
 344
 345                 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad,
 346                     partial, SHA256_BLOCK_SIZE);
 347                 if (error != 0)
 348                         return -1;
 349                 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE);
 350
 351                 break;
 352         default:
 353                 break;
 354         }
 355
 356         return 0;
 357 }
 358
 359 static inline int
 360 cipher_set_prerequisites(struct armv8_crypto_session *sess,
 361                         const struct rte_crypto_sym_xform *xform)
 362 {
 363         crypto_key_sched_t cipher_key_sched;
 364
 365         cipher_key_sched = sess->cipher.key_sched;
 366         if (likely(cipher_key_sched != NULL)) {
 367                 /* Set up cipher session key */
 368                 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data);
 369         }
 370
 371         return 0;
 372 }
 373
 374 static int
 375 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess,
 376                 const struct rte_crypto_sym_xform *cipher_xform,
 377                 const struct rte_crypto_sym_xform *auth_xform)
 378 {
 379         enum armv8_crypto_chain_order order;
 380         enum armv8_crypto_cipher_operation cop;
 381         enum rte_crypto_cipher_algorithm calg;
 382         enum rte_crypto_auth_algorithm aalg;
 383
 384         /* Validate and prepare scratch order of combined operations */
 385         switch (sess->chain_order) {
 386         case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
 387         case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
 388                 order = sess->chain_order;
 389                 break;
 390         default:
 391                 return -EINVAL;
 392         }
 393         /* Select cipher direction */
 394         sess->cipher.direction = cipher_xform->cipher.op;
 395         /* Select cipher key */
 396         sess->cipher.key.length = cipher_xform->cipher.key.length;
 397         /* Set cipher direction */
 398         cop = sess->cipher.direction;
 399         /* Set cipher algorithm */
 400         calg = cipher_xform->cipher.algo;
 401
 402         /* Select cipher algo */
 403         switch (calg) {
 404         /* Cover supported cipher algorithms */
 405         case RTE_CRYPTO_CIPHER_AES_CBC:
 406                 sess->cipher.algo = calg;
 407                 /* IV len is always 16 bytes (block size) for AES CBC */
 408                 sess->cipher.iv_len = 16;
 409                 break;
 410         default:
 411                 return -EINVAL;
 412         }
 413         /* Select auth generate/verify */
 414         sess->auth.operation = auth_xform->auth.op;
 415
 416         /* Select auth algo */
 417         switch (auth_xform->auth.algo) {
 418         /* Cover supported hash algorithms */
 419         case RTE_CRYPTO_AUTH_SHA1_HMAC:
 420         case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */
 421                 aalg = auth_xform->auth.algo;
 422                 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC;
 423                 break;
 424         default:
 425                 return -EINVAL;
 426         }
 427
 428         /* Verify supported key lengths and extract proper algorithm */
 429         switch (cipher_xform->cipher.key.length << 3) {
 430         case 128:
 431                 sess->crypto_func =
 432                                 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128);
 433                 sess->cipher.key_sched =
 434                                 CRYPTO_GET_KEY_SCHED(cop, calg, 128);
 435                 break;
 436         case 192:
 437         case 256:
 438                 /* These key lengths are not supported yet */
 439         default: /* Fall through */
 440                 sess->crypto_func = NULL;
 441                 sess->cipher.key_sched = NULL;
 442                 return -EINVAL;
 443         }
 444
 445         if (unlikely(sess->crypto_func == NULL)) {
 446                 /*
 447                  * If we got here that means that there must be a bug
 448                  * in the algorithms selection above. Nevertheless keep
 449                  * it here to catch bug immediately and avoid NULL pointer
 450                  * dereference in OPs processing.
 451                  */
 452                 ARMV8_CRYPTO_LOG_ERR(
 453                         "No appropriate crypto function for given parameters");
 454                 return -EINVAL;
 455         }
 456
 457         /* Set up cipher session prerequisites */
 458         if (cipher_set_prerequisites(sess, cipher_xform) != 0)
 459                 return -EINVAL;
 460
 461         /* Set up authentication session prerequisites */
 462         if (auth_set_prerequisites(sess, auth_xform) != 0)
 463                 return -EINVAL;
 464
 465         return 0;
 466 }
 467
 468 /** Parse crypto xform chain and set private session parameters */
 469 int
 470 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess,
 471                 const struct rte_crypto_sym_xform *xform)
 472 {
 473         const struct rte_crypto_sym_xform *cipher_xform = NULL;
 474         const struct rte_crypto_sym_xform *auth_xform = NULL;
 475         bool is_chained_op;
 476         int ret;
 477
 478         /* Filter out spurious/broken requests */
 479         if (xform == NULL)
 480                 return -EINVAL;
 481
 482         sess->chain_order = armv8_crypto_get_chain_order(xform);
 483         switch (sess->chain_order) {
 484         case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
 485                 cipher_xform = xform;
 486                 auth_xform = xform->next;
 487                 is_chained_op = true;
 488                 break;
 489         case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
 490                 auth_xform = xform;
 491                 cipher_xform = xform->next;
 492                 is_chained_op = true;
 493                 break;
 494         default:
 495                 is_chained_op = false;
 496                 return -EINVAL;
 497         }
 498
 499         if (is_chained_op) {
 500                 ret = armv8_crypto_set_session_chained_parameters(sess,
 501                                                 cipher_xform, auth_xform);
 502                 if (unlikely(ret != 0)) {
 503                         ARMV8_CRYPTO_LOG_ERR(
 504                         "Invalid/unsupported chained (cipher/auth) parameters");
 505                         return -EINVAL;
 506                 }
 507         } else {
 508                 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation");
 509                 return -EINVAL;
 510         }
 511
 512         return 0;
 513 }
 514
 515 /** Provide session for operation */
 516 static inline struct armv8_crypto_session *
 517 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
 518 {
 519         struct armv8_crypto_session *sess = NULL;
 520
 521         if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) {
 522                 /* get existing session */
 523                 if (likely(op->sym->session != NULL &&
 524                                 op->sym->session->dev_type ==
 525                                 RTE_CRYPTODEV_ARMV8_PMD)) {
 526                         sess = (struct armv8_crypto_session *)
 527                                 op->sym->session->_private;
 528                 }
 529         } else {
 530                 /* provide internal session */
 531                 void *_sess = NULL;
 532
 533                 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) {
 534                         sess = (struct armv8_crypto_session *)
 535                                 ((struct rte_cryptodev_sym_session *)_sess)
 536                                 ->_private;
 537
 538                         if (unlikely(armv8_crypto_set_session_parameters(
 539                                         sess, op->sym->xform) != 0)) {
 540                                 rte_mempool_put(qp->sess_mp, _sess);
 541                                 sess = NULL;
 542                         } else
 543                                 op->sym->session = _sess;
 544                 }
 545         }
 546
 547         if (unlikely(sess == NULL))
 548                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
 549
 550         return sess;
 551 }
 552
 553 /*
 554  *------------------------------------------------------------------------------
 555  * Process Operations
 556  *------------------------------------------------------------------------------
 557  */
 558
 559 /*----------------------------------------------------------------------------*/
 560
 561 /** Process cipher operation */
 562 static inline void
 563 process_armv8_chained_op
 564                 (struct rte_crypto_op *op, struct armv8_crypto_session *sess,
 565                 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
 566 {
 567         crypto_func_t crypto_func;
 568         crypto_arg_t arg;
 569         struct rte_mbuf *m_asrc, *m_adst;
 570         uint8_t *csrc, *cdst;
 571         uint8_t *adst, *asrc;
 572         uint64_t clen, alen;
 573         int error;
 574
 575         clen = op->sym->cipher.data.length;
 576         alen = op->sym->auth.data.length;
 577
 578         csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
 579                         op->sym->cipher.data.offset);
 580         cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
 581                         op->sym->cipher.data.offset);
 582
 583         switch (sess->chain_order) {
 584         case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
 585                 m_asrc = m_adst = mbuf_dst;
 586                 break;
 587         case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
 588                 m_asrc = mbuf_src;
 589                 m_adst = mbuf_dst;
 590                 break;
 591         default:
 592                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 593                 return;
 594         }
 595         asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *,
 596                                 op->sym->auth.data.offset);
 597
 598         switch (sess->auth.mode) {
 599         case ARMV8_CRYPTO_AUTH_AS_AUTH:
 600                 /* Nothing to do here, just verify correct option */
 601                 break;
 602         case ARMV8_CRYPTO_AUTH_AS_HMAC:
 603                 arg.digest.hmac.key = sess->auth.hmac.key;
 604                 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad;
 605                 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad;
 606                 break;
 607         default:
 608                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 609                 return;
 610         }
 611
 612         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) {
 613                 adst = op->sym->auth.digest.data;
 614                 if (adst == NULL) {
 615                         adst = rte_pktmbuf_mtod_offset(m_adst,
 616                                         uint8_t *,
 617                                         op->sym->auth.data.offset +
 618                                         op->sym->auth.data.length);
 619                 }
 620         } else {
 621                 adst = (uint8_t *)rte_pktmbuf_append(m_asrc,
 622                                 op->sym->auth.digest.length);
 623         }
 624
 625         if (unlikely(op->sym->cipher.iv.length != sess->cipher.iv_len)) {
 626                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 627                 return;
 628         }
 629
 630         arg.cipher.iv = op->sym->cipher.iv.data;
 631         arg.cipher.key = sess->cipher.key.data;
 632         /* Acquire combined mode function */
 633         crypto_func = sess->crypto_func;
 634         ARMV8_CRYPTO_ASSERT(crypto_func != NULL);
 635         error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg);
 636         if (error != 0) {
 637                 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 638                 return;
 639         }
 640
 641         op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 642         if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
 643                 if (memcmp(adst, op->sym->auth.digest.data,
 644                                 op->sym->auth.digest.length) != 0) {
 645                         op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
 646                 }
 647                 /* Trim area used for digest from mbuf. */
 648                 rte_pktmbuf_trim(m_asrc,
 649                                 op->sym->auth.digest.length);
 650         }
 651 }
 652
 653 /** Process crypto operation for mbuf */
 654 static inline int
 655 process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
 656                 struct armv8_crypto_session *sess)
 657 {
 658         struct rte_mbuf *msrc, *mdst;
 659
 660         msrc = op->sym->m_src;
 661         mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
 662
 663         op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
 664
 665         switch (sess->chain_order) {
 666         case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
 667         case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */
 668                 process_armv8_chained_op(op, sess, msrc, mdst);
 669                 break;
 670         default:
 671                 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
 672                 break;
 673         }
 674
 675         /* Free session if a session-less crypto op */
 676         if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) {
 677                 memset(sess, 0, sizeof(struct armv8_crypto_session));
 678                 rte_mempool_put(qp->sess_mp, op->sym->session);
 679                 op->sym->session = NULL;
 680         }
 681
 682         if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
 683                 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
 684
 685         if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR))
 686                 return -1;
 687
 688         return 0;
 689 }
 690
 691 /*
 692  *------------------------------------------------------------------------------
 693  * PMD Framework
 694  *------------------------------------------------------------------------------
 695  */
 696
 697 /** Enqueue burst */
 698 static uint16_t
 699 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
 700                 uint16_t nb_ops)
 701 {
 702         struct armv8_crypto_session *sess;
 703         struct armv8_crypto_qp *qp = queue_pair;
 704         int i, retval;
 705
 706         for (i = 0; i < nb_ops; i++) {
 707                 sess = get_session(qp, ops[i]);
 708                 if (unlikely(sess == NULL))
 709                         goto enqueue_err;
 710
 711                 retval = process_op(qp, ops[i], sess);
 712                 if (unlikely(retval < 0))
 713                         goto enqueue_err;
 714         }
 715
 716         retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
 717                         NULL);
 718         qp->stats.enqueued_count += retval;
 719
 720         return retval;
 721
 722 enqueue_err:
 723         retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
 724                         NULL);
 725         if (ops[i] != NULL)
 726                 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
 727
 728         qp->stats.enqueue_err_count++;
 729         return retval;
 730 }
 731
 732 /** Dequeue burst */
 733 static uint16_t
 734 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
 735                 uint16_t nb_ops)
 736 {
 737         struct armv8_crypto_qp *qp = queue_pair;
 738
 739         unsigned int nb_dequeued = 0;
 740
 741         nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
 742                         (void **)ops, nb_ops, NULL);
 743         qp->stats.dequeued_count += nb_dequeued;
 744
 745         return nb_dequeued;
 746 }
 747
 748 /** Create ARMv8 crypto device */
 749 static int
 750 cryptodev_armv8_crypto_create(const char *name,
 751                         struct rte_vdev_device *vdev,
 752                         struct rte_crypto_vdev_init_params *init_params)
 753 {
 754         struct rte_cryptodev *dev;
 755         struct armv8_crypto_private *internals;
 756
 757         /* Check CPU for support for AES instruction set */
 758         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
 759                 ARMV8_CRYPTO_LOG_ERR(
 760                         "AES instructions not supported by CPU");
 761                 return -EFAULT;
 762         }
 763
 764         /* Check CPU for support for SHA instruction set */
 765         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) ||
 766             !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) {
 767                 ARMV8_CRYPTO_LOG_ERR(
 768                         "SHA1/SHA2 instructions not supported by CPU");
 769                 return -EFAULT;
 770         }
 771
 772         /* Check CPU for support for Advance SIMD instruction set */
 773         if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
 774                 ARMV8_CRYPTO_LOG_ERR(
 775                         "Advanced SIMD instructions not supported by CPU");
 776                 return -EFAULT;
 777         }
 778
 779         if (init_params->name[0] == '\0')
 780                 snprintf(init_params->name, sizeof(init_params->name),
 781                                 "%s", name);
 782
 783         dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name,
 784                                 sizeof(struct armv8_crypto_private),
 785                                 init_params->socket_id);
 786         if (dev == NULL) {
 787                 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
 788                 goto init_error;
 789         }
 790
 791         dev->dev_type = RTE_CRYPTODEV_ARMV8_PMD;
 792         dev->dev_ops = rte_armv8_crypto_pmd_ops;
 793
 794         /* register rx/tx burst functions for data path */
 795         dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst;
 796         dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst;
 797
 798         dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
 799                         RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
 800                         RTE_CRYPTODEV_FF_CPU_NEON |
 801                         RTE_CRYPTODEV_FF_CPU_ARM_CE;
 802
 803         /* Set vector instructions mode supported */
 804         internals = dev->data->dev_private;
 805
 806         internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
 807         internals->max_nb_sessions = init_params->max_nb_sessions;
 808
 809         return 0;
 810
 811 init_error:
 812         ARMV8_CRYPTO_LOG_ERR(
 813                 "driver %s: cryptodev_armv8_crypto_create failed",
 814                 init_params->name);
 815
 816         cryptodev_armv8_crypto_uninit(vdev);
 817         return -EFAULT;
 818 }
 819
 820 /** Initialise ARMv8 crypto device */
 821 static int
 822 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev)
 823 {
 824         struct rte_crypto_vdev_init_params init_params = {
 825                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
 826                 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
 827                 rte_socket_id(),
 828                 {0}
 829         };
 830         const char *name;
 831         const char *input_args;
 832
 833         name = rte_vdev_device_name(vdev);
 834         if (name == NULL)
 835                 return -EINVAL;
 836         input_args = rte_vdev_device_args(vdev);
 837         rte_cryptodev_parse_vdev_init_params(&init_params, input_args);
 838
 839         RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
 840                         init_params.socket_id);
 841         if (init_params.name[0] != '\0') {
 842                 RTE_LOG(INFO, PMD, "  User defined name = %s\n",
 843                         init_params.name);
 844         }
 845         RTE_LOG(INFO, PMD, "  Max number of queue pairs = %d\n",
 846                         init_params.max_nb_queue_pairs);
 847         RTE_LOG(INFO, PMD, "  Max number of sessions = %d\n",
 848                         init_params.max_nb_sessions);
 849
 850         return cryptodev_armv8_crypto_create(name, vdev, &init_params);
 851 }
 852
 853 /** Uninitialise ARMv8 crypto device */
 854 static int
 855 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev)
 856 {
 857         const char *name;
 858
 859         name = rte_vdev_device_name(vdev);
 860         if (name == NULL)
 861                 return -EINVAL;
 862
 863         RTE_LOG(INFO, PMD,
 864                 "Closing ARMv8 crypto device %s on numa socket %u\n",
 865                 name, rte_socket_id());
 866
 867         return 0;
 868 }
 869
 870 static struct rte_vdev_driver armv8_crypto_drv = {
 871         .probe = cryptodev_armv8_crypto_init,
 872         .remove = cryptodev_armv8_crypto_uninit
 873 };
 874
 875 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_drv);
 876 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd);
 877 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD,
 878         "max_nb_queue_pairs=<int> "
 879         "max_nb_sessions=<int> "
 880         "socket_id=<int>");