4 * Copyright (C) Cavium networks Ltd. 2017.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Cavium networks nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 #include <rte_common.h>
36 #include <rte_hexdump.h>
37 #include <rte_cryptodev.h>
38 #include <rte_cryptodev_pmd.h>
40 #include <rte_malloc.h>
41 #include <rte_cpuflags.h>
43 #include "armv8_crypto_defs.h"
45 #include "rte_armv8_pmd_private.h"
47 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev);
50 * Pointers to the supported combined mode crypto functions are stored
51 * in the static tables. Each combined (chained) cryptographic operation
52 * can be described by a set of numbers:
53 * - order: order of operations (cipher, auth) or (auth, cipher)
54 * - direction: encryption or decryption
55 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc.
56 * - aalg: authentication algorithm such as SHA1, SHA256, etc.
57 * - keyl: cipher key length, for example 128, 192, 256 bits
59 * In order to quickly acquire each function pointer based on those numbers,
60 * a hierarchy of arrays is maintained. The final level, 3D array is indexed
61 * by the combined mode function parameters only (cipher algorithm,
62 * authentication algorithm and key length).
64 * This gives 3 memory accesses to obtain a function pointer instead of
65 * traversing the array manually and comparing function parameters on each loop.
79 * 3D array type for ARM Combined Mode crypto functions pointers.
80 * CRYPTO_CIPHER_MAX: max cipher ID number
81 * CRYPTO_AUTH_MAX: max auth ID number
82 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
84 typedef const crypto_func_t
85 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
87 /* Evaluate to key length definition */
88 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl)
90 /* Local aliases for supported ciphers */
91 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC
92 /* Local aliases for supported hashes */
93 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC
94 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC
97 * Arrays containing pointers to particular cryptographic,
98 * combined mode functions.
99 * crypto_op_ca_encrypt: cipher (encrypt), authenticate
100 * crypto_op_ca_decrypt: cipher (decrypt), authenticate
101 * crypto_op_ac_encrypt: authenticate, cipher (encrypt)
102 * crypto_op_ac_decrypt: authenticate, cipher (decrypt)
104 static const crypto_func_tbl_t
105 crypto_op_ca_encrypt = {
106 /* [cipher alg][auth alg][key length] = crypto_function, */
107 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac,
108 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac,
111 static const crypto_func_tbl_t
112 crypto_op_ca_decrypt = {
116 static const crypto_func_tbl_t
117 crypto_op_ac_encrypt = {
121 static const crypto_func_tbl_t
122 crypto_op_ac_decrypt = {
123 /* [cipher alg][auth alg][key length] = crypto_function, */
124 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec,
125 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec,
129 * Arrays containing pointers to particular cryptographic function sets,
130 * covering given cipher operation directions (encrypt, decrypt)
131 * for each order of cipher and authentication pairs.
133 static const crypto_func_tbl_t *
134 crypto_cipher_auth[] = {
135 &crypto_op_ca_encrypt,
136 &crypto_op_ca_decrypt,
140 static const crypto_func_tbl_t *
141 crypto_auth_cipher[] = {
142 &crypto_op_ac_encrypt,
143 &crypto_op_ac_decrypt,
148 * Top level array containing pointers to particular cryptographic
149 * function sets, covering given order of chained operations.
150 * crypto_cipher_auth: cipher first, authenticate after
151 * crypto_auth_cipher: authenticate first, cipher after
153 static const crypto_func_tbl_t **
154 crypto_chain_order[] = {
161 * Extract particular combined mode crypto function from the 3D array.
163 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \
165 crypto_func_tbl_t *func_tbl = \
166 (crypto_chain_order[(order)])[(cop)]; \
168 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \
171 /*----------------------------------------------------------------------------*/
174 * 2D array type for ARM key schedule functions pointers.
175 * CRYPTO_CIPHER_MAX: max cipher ID number
176 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number
178 typedef const crypto_key_sched_t
179 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX];
181 static const crypto_key_sched_tbl_t
182 crypto_key_sched_encrypt = {
183 /* [cipher alg][key length] = key_expand_func, */
184 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc,
187 static const crypto_key_sched_tbl_t
188 crypto_key_sched_decrypt = {
189 /* [cipher alg][key length] = key_expand_func, */
190 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec,
194 * Top level array containing pointers to particular key generation
195 * function sets, covering given operation direction.
196 * crypto_key_sched_encrypt: keys for encryption
197 * crypto_key_sched_decrypt: keys for decryption
199 static const crypto_key_sched_tbl_t *
200 crypto_key_sched_dir[] = {
201 &crypto_key_sched_encrypt,
202 &crypto_key_sched_decrypt,
207 * Extract particular combined mode crypto function from the 3D array.
209 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \
211 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \
213 ((*ks_tbl)[(calg)][KEYL(keyl)]); \
216 /*----------------------------------------------------------------------------*/
219 *------------------------------------------------------------------------------
221 *------------------------------------------------------------------------------
224 /** Get xform chain order */
225 static enum armv8_crypto_chain_order
226 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform)
230 * This driver currently covers only chained operations.
231 * Ignore only cipher or only authentication operations
232 * or chains longer than 2 xform structures.
234 if (xform->next == NULL || xform->next->next != NULL)
235 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
237 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) {
238 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER)
239 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER;
242 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) {
243 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH)
244 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH;
247 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED;
251 auth_hmac_pad_prepare(struct armv8_crypto_session *sess,
252 const struct rte_crypto_sym_xform *xform)
256 /* Generate i_key_pad and o_key_pad */
257 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad));
258 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key,
259 xform->auth.key.length);
260 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad));
261 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key,
262 xform->auth.key.length);
264 * XOR key with IPAD/OPAD values to obtain i_key_pad
266 * Byte-by-byte operation may seem to be the less efficient
267 * here but in fact it's the opposite.
268 * The result ASM code is likely operate on NEON registers
269 * (load auth key to Qx, load IPAD/OPAD to multiple
270 * elements of Qy, eor 128 bits at once).
272 for (i = 0; i < SHA_BLOCK_MAX; i++) {
273 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE;
274 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE;
279 auth_set_prerequisites(struct armv8_crypto_session *sess,
280 const struct rte_crypto_sym_xform *xform)
282 uint8_t partial[64] = { 0 };
285 switch (xform->auth.algo) {
286 case RTE_CRYPTO_AUTH_SHA1_HMAC:
288 * Generate authentication key, i_key_pad and o_key_pad.
290 /* Zero memory under key */
291 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE);
294 * Now copy the given authentication key to the session
297 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
298 xform->auth.key.length);
300 /* Prepare HMAC padding: key|pattern */
301 auth_hmac_pad_prepare(sess, xform);
303 * Calculate partial hash values for i_key_pad and o_key_pad.
304 * Will be used as initialization state for final HMAC.
306 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad,
307 partial, SHA1_BLOCK_SIZE);
310 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE);
312 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad,
313 partial, SHA1_BLOCK_SIZE);
316 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE);
319 case RTE_CRYPTO_AUTH_SHA256_HMAC:
321 * Generate authentication key, i_key_pad and o_key_pad.
323 /* Zero memory under key */
324 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE);
327 * Now copy the given authentication key to the session
330 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data,
331 xform->auth.key.length);
333 /* Prepare HMAC padding: key|pattern */
334 auth_hmac_pad_prepare(sess, xform);
336 * Calculate partial hash values for i_key_pad and o_key_pad.
337 * Will be used as initialization state for final HMAC.
339 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad,
340 partial, SHA256_BLOCK_SIZE);
343 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE);
345 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad,
346 partial, SHA256_BLOCK_SIZE);
349 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE);
360 cipher_set_prerequisites(struct armv8_crypto_session *sess,
361 const struct rte_crypto_sym_xform *xform)
363 crypto_key_sched_t cipher_key_sched;
365 cipher_key_sched = sess->cipher.key_sched;
366 if (likely(cipher_key_sched != NULL)) {
367 /* Set up cipher session key */
368 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data);
375 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess,
376 const struct rte_crypto_sym_xform *cipher_xform,
377 const struct rte_crypto_sym_xform *auth_xform)
379 enum armv8_crypto_chain_order order;
380 enum armv8_crypto_cipher_operation cop;
381 enum rte_crypto_cipher_algorithm calg;
382 enum rte_crypto_auth_algorithm aalg;
384 /* Validate and prepare scratch order of combined operations */
385 switch (sess->chain_order) {
386 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
387 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
388 order = sess->chain_order;
393 /* Select cipher direction */
394 sess->cipher.direction = cipher_xform->cipher.op;
395 /* Select cipher key */
396 sess->cipher.key.length = cipher_xform->cipher.key.length;
397 /* Set cipher direction */
398 cop = sess->cipher.direction;
399 /* Set cipher algorithm */
400 calg = cipher_xform->cipher.algo;
402 /* Select cipher algo */
404 /* Cover supported cipher algorithms */
405 case RTE_CRYPTO_CIPHER_AES_CBC:
406 sess->cipher.algo = calg;
407 /* IV len is always 16 bytes (block size) for AES CBC */
408 sess->cipher.iv_len = 16;
413 /* Select auth generate/verify */
414 sess->auth.operation = auth_xform->auth.op;
416 /* Select auth algo */
417 switch (auth_xform->auth.algo) {
418 /* Cover supported hash algorithms */
419 case RTE_CRYPTO_AUTH_SHA1_HMAC:
420 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */
421 aalg = auth_xform->auth.algo;
422 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC;
428 /* Verify supported key lengths and extract proper algorithm */
429 switch (cipher_xform->cipher.key.length << 3) {
432 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128);
433 sess->cipher.key_sched =
434 CRYPTO_GET_KEY_SCHED(cop, calg, 128);
438 /* These key lengths are not supported yet */
439 default: /* Fall through */
440 sess->crypto_func = NULL;
441 sess->cipher.key_sched = NULL;
445 if (unlikely(sess->crypto_func == NULL)) {
447 * If we got here that means that there must be a bug
448 * in the algorithms selection above. Nevertheless keep
449 * it here to catch bug immediately and avoid NULL pointer
450 * dereference in OPs processing.
452 ARMV8_CRYPTO_LOG_ERR(
453 "No appropriate crypto function for given parameters");
457 /* Set up cipher session prerequisites */
458 if (cipher_set_prerequisites(sess, cipher_xform) != 0)
461 /* Set up authentication session prerequisites */
462 if (auth_set_prerequisites(sess, auth_xform) != 0)
468 /** Parse crypto xform chain and set private session parameters */
470 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess,
471 const struct rte_crypto_sym_xform *xform)
473 const struct rte_crypto_sym_xform *cipher_xform = NULL;
474 const struct rte_crypto_sym_xform *auth_xform = NULL;
478 /* Filter out spurious/broken requests */
482 sess->chain_order = armv8_crypto_get_chain_order(xform);
483 switch (sess->chain_order) {
484 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
485 cipher_xform = xform;
486 auth_xform = xform->next;
487 is_chained_op = true;
489 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
491 cipher_xform = xform->next;
492 is_chained_op = true;
495 is_chained_op = false;
500 ret = armv8_crypto_set_session_chained_parameters(sess,
501 cipher_xform, auth_xform);
502 if (unlikely(ret != 0)) {
503 ARMV8_CRYPTO_LOG_ERR(
504 "Invalid/unsupported chained (cipher/auth) parameters");
508 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation");
515 /** Provide session for operation */
516 static inline struct armv8_crypto_session *
517 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op)
519 struct armv8_crypto_session *sess = NULL;
521 if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_WITH_SESSION) {
522 /* get existing session */
523 if (likely(op->sym->session != NULL &&
524 op->sym->session->dev_type ==
525 RTE_CRYPTODEV_ARMV8_PMD)) {
526 sess = (struct armv8_crypto_session *)
527 op->sym->session->_private;
530 /* provide internal session */
533 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) {
534 sess = (struct armv8_crypto_session *)
535 ((struct rte_cryptodev_sym_session *)_sess)
538 if (unlikely(armv8_crypto_set_session_parameters(
539 sess, op->sym->xform) != 0)) {
540 rte_mempool_put(qp->sess_mp, _sess);
543 op->sym->session = _sess;
547 if (unlikely(sess == NULL))
548 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION;
554 *------------------------------------------------------------------------------
556 *------------------------------------------------------------------------------
559 /*----------------------------------------------------------------------------*/
561 /** Process cipher operation */
563 process_armv8_chained_op
564 (struct rte_crypto_op *op, struct armv8_crypto_session *sess,
565 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst)
567 crypto_func_t crypto_func;
569 struct rte_mbuf *m_asrc, *m_adst;
570 uint8_t *csrc, *cdst;
571 uint8_t *adst, *asrc;
575 clen = op->sym->cipher.data.length;
576 alen = op->sym->auth.data.length;
578 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *,
579 op->sym->cipher.data.offset);
580 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *,
581 op->sym->cipher.data.offset);
583 switch (sess->chain_order) {
584 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
585 m_asrc = m_adst = mbuf_dst;
587 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER:
592 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
595 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *,
596 op->sym->auth.data.offset);
598 switch (sess->auth.mode) {
599 case ARMV8_CRYPTO_AUTH_AS_AUTH:
600 /* Nothing to do here, just verify correct option */
602 case ARMV8_CRYPTO_AUTH_AS_HMAC:
603 arg.digest.hmac.key = sess->auth.hmac.key;
604 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad;
605 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad;
608 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
612 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) {
613 adst = op->sym->auth.digest.data;
615 adst = rte_pktmbuf_mtod_offset(m_adst,
617 op->sym->auth.data.offset +
618 op->sym->auth.data.length);
621 adst = (uint8_t *)rte_pktmbuf_append(m_asrc,
622 op->sym->auth.digest.length);
625 if (unlikely(op->sym->cipher.iv.length != sess->cipher.iv_len)) {
626 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
630 arg.cipher.iv = op->sym->cipher.iv.data;
631 arg.cipher.key = sess->cipher.key.data;
632 /* Acquire combined mode function */
633 crypto_func = sess->crypto_func;
634 ARMV8_CRYPTO_ASSERT(crypto_func != NULL);
635 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg);
637 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
641 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
642 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) {
643 if (memcmp(adst, op->sym->auth.digest.data,
644 op->sym->auth.digest.length) != 0) {
645 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED;
647 /* Trim area used for digest from mbuf. */
648 rte_pktmbuf_trim(m_asrc,
649 op->sym->auth.digest.length);
653 /** Process crypto operation for mbuf */
655 process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op,
656 struct armv8_crypto_session *sess)
658 struct rte_mbuf *msrc, *mdst;
660 msrc = op->sym->m_src;
661 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src;
663 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED;
665 switch (sess->chain_order) {
666 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH:
667 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */
668 process_armv8_chained_op(op, sess, msrc, mdst);
671 op->status = RTE_CRYPTO_OP_STATUS_ERROR;
675 /* Free session if a session-less crypto op */
676 if (op->sym->sess_type == RTE_CRYPTO_SYM_OP_SESSIONLESS) {
677 memset(sess, 0, sizeof(struct armv8_crypto_session));
678 rte_mempool_put(qp->sess_mp, op->sym->session);
679 op->sym->session = NULL;
682 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED)
683 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS;
685 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR))
692 *------------------------------------------------------------------------------
694 *------------------------------------------------------------------------------
699 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops,
702 struct armv8_crypto_session *sess;
703 struct armv8_crypto_qp *qp = queue_pair;
706 for (i = 0; i < nb_ops; i++) {
707 sess = get_session(qp, ops[i]);
708 if (unlikely(sess == NULL))
711 retval = process_op(qp, ops[i], sess);
712 if (unlikely(retval < 0))
716 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
718 qp->stats.enqueued_count += retval;
723 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i,
726 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS;
728 qp->stats.enqueue_err_count++;
734 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops,
737 struct armv8_crypto_qp *qp = queue_pair;
739 unsigned int nb_dequeued = 0;
741 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops,
742 (void **)ops, nb_ops, NULL);
743 qp->stats.dequeued_count += nb_dequeued;
748 /** Create ARMv8 crypto device */
750 cryptodev_armv8_crypto_create(const char *name,
751 struct rte_vdev_device *vdev,
752 struct rte_crypto_vdev_init_params *init_params)
754 struct rte_cryptodev *dev;
755 struct armv8_crypto_private *internals;
757 /* Check CPU for support for AES instruction set */
758 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) {
759 ARMV8_CRYPTO_LOG_ERR(
760 "AES instructions not supported by CPU");
764 /* Check CPU for support for SHA instruction set */
765 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) ||
766 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) {
767 ARMV8_CRYPTO_LOG_ERR(
768 "SHA1/SHA2 instructions not supported by CPU");
772 /* Check CPU for support for Advance SIMD instruction set */
773 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) {
774 ARMV8_CRYPTO_LOG_ERR(
775 "Advanced SIMD instructions not supported by CPU");
779 if (init_params->name[0] == '\0')
780 snprintf(init_params->name, sizeof(init_params->name),
783 dev = rte_cryptodev_pmd_virtual_dev_init(init_params->name,
784 sizeof(struct armv8_crypto_private),
785 init_params->socket_id);
787 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev");
791 dev->dev_type = RTE_CRYPTODEV_ARMV8_PMD;
792 dev->dev_ops = rte_armv8_crypto_pmd_ops;
794 /* register rx/tx burst functions for data path */
795 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst;
796 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst;
798 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO |
799 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING |
800 RTE_CRYPTODEV_FF_CPU_NEON |
801 RTE_CRYPTODEV_FF_CPU_ARM_CE;
803 /* Set vector instructions mode supported */
804 internals = dev->data->dev_private;
806 internals->max_nb_qpairs = init_params->max_nb_queue_pairs;
807 internals->max_nb_sessions = init_params->max_nb_sessions;
812 ARMV8_CRYPTO_LOG_ERR(
813 "driver %s: cryptodev_armv8_crypto_create failed",
816 cryptodev_armv8_crypto_uninit(vdev);
820 /** Initialise ARMv8 crypto device */
822 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev)
824 struct rte_crypto_vdev_init_params init_params = {
825 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS,
826 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS,
831 const char *input_args;
833 name = rte_vdev_device_name(vdev);
836 input_args = rte_vdev_device_args(vdev);
837 rte_cryptodev_parse_vdev_init_params(&init_params, input_args);
839 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name,
840 init_params.socket_id);
841 if (init_params.name[0] != '\0') {
842 RTE_LOG(INFO, PMD, " User defined name = %s\n",
845 RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n",
846 init_params.max_nb_queue_pairs);
847 RTE_LOG(INFO, PMD, " Max number of sessions = %d\n",
848 init_params.max_nb_sessions);
850 return cryptodev_armv8_crypto_create(name, vdev, &init_params);
853 /** Uninitialise ARMv8 crypto device */
855 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev)
859 name = rte_vdev_device_name(vdev);
864 "Closing ARMv8 crypto device %s on numa socket %u\n",
865 name, rte_socket_id());
870 static struct rte_vdev_driver armv8_crypto_drv = {
871 .probe = cryptodev_armv8_crypto_init,
872 .remove = cryptodev_armv8_crypto_uninit
875 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_drv);
876 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd);
877 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD,
878 "max_nb_queue_pairs=<int> "
879 "max_nb_sessions=<int> "