2 *------------------------------------------------------------------
3 * Copyright (c) 2020 Intel and/or its affiliates.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 *------------------------------------------------------------------
18 #include <vlib/vlib.h>
19 #include <vnet/plugin/plugin.h>
20 #include <vnet/crypto/crypto.h>
21 #include <vnet/ipsec/ipsec.h>
22 #include <vpp/app/version.h>
24 #include <dpdk/buffer.h>
25 #include <dpdk/device/dpdk.h>
26 #include <dpdk/device/dpdk_priv.h>
28 #include <rte_bus_vdev.h>
29 #include <rte_cryptodev.h>
30 #include <rte_crypto_sym.h>
31 #include <rte_crypto.h>
32 #include <rte_malloc.h>
33 #include <rte_config.h>
35 #include "cryptodev.h"
38 #define always_inline static inline
40 #define always_inline static inline __attribute__ ((__always_inline__))
43 static_always_inline u64
44 compute_ofs_linked_alg (vnet_crypto_async_frame_elt_t *fe, i16 *min_ofs,
47 union rte_crypto_sym_ofs ofs;
48 u32 crypto_end = fe->crypto_start_offset + fe->crypto_total_length;
50 fe->integ_start_offset + fe->crypto_total_length + fe->integ_length_adj;
52 *min_ofs = clib_min (fe->crypto_start_offset, fe->integ_start_offset);
53 *max_end = clib_max (crypto_end, integ_end);
55 ofs.ofs.cipher.head = fe->crypto_start_offset - *min_ofs;
56 ofs.ofs.cipher.tail = *max_end - crypto_end;
57 ofs.ofs.auth.head = fe->integ_start_offset - *min_ofs;
58 ofs.ofs.auth.tail = *max_end - integ_end;
63 static_always_inline int
64 cryptodev_frame_build_sgl (vlib_main_t *vm, enum rte_iova_mode iova_mode,
65 struct rte_crypto_vec *data_vec, u16 *n_seg,
66 vlib_buffer_t *b, u32 size)
68 struct rte_crypto_vec *vec = data_vec + 1;
69 if (vlib_buffer_chain_linearize (vm, b) > CRYPTODEV_MAX_N_SGL)
72 while ((b->flags & VLIB_BUFFER_NEXT_PRESENT) && size)
75 b = vlib_get_buffer (vm, b->next_buffer);
76 len = clib_min (b->current_length, size);
77 vec->base = (void *) vlib_buffer_get_current (b);
78 if (iova_mode == RTE_IOVA_VA)
79 vec->iova = pointer_to_uword (vec->base);
81 vec->iova = vlib_buffer_get_current_pa (vm, b);
94 static_always_inline void
95 cryptodev_reset_ctx (cryptodev_engine_thread_t *cet)
97 union rte_cryptodev_session_ctx sess_ctx;
99 ASSERT (cet->reset_sess != 0);
101 sess_ctx.crypto_sess = cet->reset_sess;
103 rte_cryptodev_configure_raw_dp_ctx (cet->cryptodev_id, cet->cryptodev_q,
104 cet->ctx, RTE_CRYPTO_OP_WITH_SESSION,
108 static_always_inline int
109 cryptodev_frame_linked_algs_enqueue (vlib_main_t *vm,
110 vnet_crypto_async_frame_t *frame,
111 cryptodev_op_type_t op_type)
113 cryptodev_main_t *cmt = &cryptodev_main;
114 cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
115 vnet_crypto_async_frame_elt_t *fe;
117 struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL];
118 struct rte_crypto_va_iova_ptr iv_vec, digest_vec;
120 u32 last_key_index = ~0;
126 n_elts = frame->n_elts;
128 if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts))
130 cryptodev_mark_frame_err_status (frame,
131 VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
135 vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts);
142 union rte_crypto_sym_ofs cofs;
147 clib_prefetch_load (&fe[1]);
148 clib_prefetch_load (&fe[2]);
149 vlib_prefetch_buffer_header (b[1], LOAD);
150 vlib_prefetch_buffer_header (b[2], LOAD);
153 if (PREDICT_FALSE (last_key_index != fe->key_index))
155 cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
156 union rte_cryptodev_session_ctx sess_ctx;
158 if (PREDICT_FALSE (key->keys[vm->numa_node][op_type] == 0))
160 status = cryptodev_session_create (vm, fe->key_index, 0);
161 if (PREDICT_FALSE (status < 0))
165 /* Borrow a created session to reset session ctx, based on a valid
166 * assumption that the session reset won't happen until first valid
167 * packet is processed */
168 if (PREDICT_FALSE (cet->reset_sess == 0))
169 cet->reset_sess = key->keys[vm->numa_node][op_type];
171 sess_ctx.crypto_sess = key->keys[vm->numa_node][op_type];
173 status = rte_cryptodev_configure_raw_dp_ctx (
174 cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
175 RTE_CRYPTO_OP_WITH_SESSION, sess_ctx, is_update);
176 if (PREDICT_FALSE (status < 0))
179 last_key_index = fe->key_index;
183 cofs.raw = compute_ofs_linked_alg (fe, &min_ofs, &max_end);
185 vec->len = max_end - min_ofs;
186 if (cmt->iova_mode == RTE_IOVA_VA)
188 vec[0].base = (void *) (b[0]->data + min_ofs);
189 vec[0].iova = pointer_to_uword (b[0]->data) + min_ofs;
190 iv_vec.va = (void *) fe->iv;
191 iv_vec.iova = pointer_to_uword (fe->iv);
192 digest_vec.va = (void *) fe->tag;
193 digest_vec.iova = pointer_to_uword (fe->tag);
197 vec[0].base = (void *) (b[0]->data + min_ofs);
198 vec[0].iova = vlib_buffer_get_pa (vm, b[0]) + min_ofs;
199 iv_vec.va = (void *) fe->iv;
200 iv_vec.iova = vlib_physmem_get_pa (vm, fe->iv);
201 digest_vec.va = (void *) fe->tag;
202 digest_vec.iova = vlib_physmem_get_pa (vm, fe->digest);
205 if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS))
207 vec[0].len = b[0]->current_data + b[0]->current_length - min_ofs;
208 if (cryptodev_frame_build_sgl (vm, cmt->iova_mode, vec, &n_seg, b[0],
209 max_end - min_ofs - vec->len) < 0)
213 status = rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs, &iv_vec,
214 &digest_vec, 0, (void *) frame);
215 if (PREDICT_FALSE (status < 0))
223 status = rte_cryptodev_raw_enqueue_done (cet->ctx, frame->n_elts);
224 if (PREDICT_FALSE (status < 0))
226 cryptodev_reset_ctx (cet);
230 cet->inflight += frame->n_elts;
234 cryptodev_mark_frame_err_status (frame,
235 VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
236 cryptodev_reset_ctx (cet);
240 static_always_inline int
241 cryptodev_raw_aead_enqueue (vlib_main_t *vm, vnet_crypto_async_frame_t *frame,
242 cryptodev_op_type_t op_type, u8 aad_len)
244 cryptodev_main_t *cmt = &cryptodev_main;
245 cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
246 vnet_crypto_async_frame_elt_t *fe;
249 union rte_crypto_sym_ofs cofs;
250 struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL];
251 struct rte_crypto_va_iova_ptr iv_vec, digest_vec, aad_vec;
252 u32 last_key_index = ~0;
256 n_elts = frame->n_elts;
258 if (PREDICT_FALSE (CRYPTODEV_MAX_INFLIGHT - cet->inflight < n_elts))
260 cryptodev_mark_frame_err_status (frame,
261 VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
265 vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts);
273 u32 aad_offset = ((cet->aad_index++) & CRYPTODEV_AAD_MASK) << 4;
278 clib_prefetch_load (&fe[1]);
279 vlib_prefetch_buffer_header (b[1], LOAD);
282 if (PREDICT_FALSE (last_key_index != fe->key_index))
284 cryptodev_key_t *key = vec_elt_at_index (cmt->keys, fe->key_index);
285 union rte_cryptodev_session_ctx sess_ctx;
287 if (PREDICT_FALSE (key->keys[vm->numa_node][op_type] == 0))
289 status = cryptodev_session_create (vm, fe->key_index, aad_len);
290 if (PREDICT_FALSE (status < 0))
295 #if RTE_VERSION >= RTE_VERSION_NUM(22, 11, 0, 0)
296 rte_cryptodev_sym_session_opaque_data_get (
297 key->keys[vm->numa_node][op_type]) != (u64) aad_len
299 (u8) key->keys[vm->numa_node][op_type]->opaque_data != aad_len
303 cryptodev_sess_handler (vm, VNET_CRYPTO_KEY_OP_DEL,
304 fe->key_index, aad_len);
305 status = cryptodev_session_create (vm, fe->key_index, aad_len);
306 if (PREDICT_FALSE (status < 0))
310 /* Borrow a created session to reset session ctx, based on a valid
311 * assumption that the session reset won't happen until first valid
312 * packet is processed */
314 if (PREDICT_FALSE (cet->reset_sess == 0))
315 cet->reset_sess = key->keys[vm->numa_node][op_type];
317 sess_ctx.crypto_sess = key->keys[vm->numa_node][op_type];
319 status = rte_cryptodev_configure_raw_dp_ctx (
320 cet->cryptodev_id, cet->cryptodev_q, cet->ctx,
321 RTE_CRYPTO_OP_WITH_SESSION, sess_ctx, is_update);
322 if (PREDICT_FALSE (status < 0))
325 last_key_index = fe->key_index;
329 if (cmt->iova_mode == RTE_IOVA_VA)
331 vec[0].base = (void *) (b[0]->data + fe->crypto_start_offset);
332 vec[0].iova = pointer_to_uword (vec[0].base);
333 vec[0].len = fe->crypto_total_length;
334 iv_vec.va = (void *) fe->iv;
335 iv_vec.iova = pointer_to_uword (fe->iv);
336 digest_vec.va = (void *) fe->tag;
337 digest_vec.iova = pointer_to_uword (fe->tag);
338 aad_vec.va = (void *) (cet->aad_buf + aad_offset);
339 aad_vec.iova = cet->aad_phy_addr + aad_offset;
343 vec[0].base = (void *) (b[0]->data + fe->crypto_start_offset);
345 vlib_buffer_get_pa (vm, b[0]) + fe->crypto_start_offset;
346 vec[0].len = fe->crypto_total_length;
347 iv_vec.va = (void *) fe->iv;
348 iv_vec.iova = vlib_physmem_get_pa (vm, fe->iv);
349 aad_vec.va = (void *) (cet->aad_buf + aad_offset);
350 aad_vec.iova = cet->aad_phy_addr + aad_offset;
351 digest_vec.va = (void *) fe->tag;
352 digest_vec.iova = vlib_physmem_get_pa (vm, fe->tag);
356 *(u64 *) (cet->aad_buf + aad_offset) = *(u64 *) fe->aad;
357 else if (aad_len != 0)
360 *(u64 *) (cet->aad_buf + aad_offset) = *(u64 *) fe->aad;
361 *(u32 *) (cet->aad_buf + aad_offset + 8) = *(u32 *) (fe->aad + 8);
364 if (PREDICT_FALSE (fe->flags & VNET_CRYPTO_OP_FLAG_CHAINED_BUFFERS))
366 vec[0].len = b[0]->current_data + b[0]->current_length -
367 fe->crypto_start_offset;
369 cryptodev_frame_build_sgl (vm, cmt->iova_mode, vec, &n_seg, b[0],
370 fe->crypto_total_length - vec[0].len);
376 rte_cryptodev_raw_enqueue (cet->ctx, vec, n_seg, cofs, &iv_vec,
377 &digest_vec, &aad_vec, (void *) frame);
378 if (PREDICT_FALSE (status < 0))
386 status = rte_cryptodev_raw_enqueue_done (cet->ctx, frame->n_elts);
387 if (PREDICT_FALSE (status < 0))
390 cet->inflight += frame->n_elts;
395 cryptodev_mark_frame_err_status (frame,
396 VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
397 cryptodev_reset_ctx (cet);
401 static_always_inline u32
402 cryptodev_get_frame_n_elts (void *frame)
404 vnet_crypto_async_frame_t *f = (vnet_crypto_async_frame_t *) frame;
408 static_always_inline void
409 cryptodev_post_dequeue (void *frame, u32 index, u8 is_op_success)
411 vnet_crypto_async_frame_t *f = (vnet_crypto_async_frame_t *) frame;
413 f->elts[index].status = is_op_success ? VNET_CRYPTO_OP_STATUS_COMPLETED :
414 VNET_CRYPTO_OP_STATUS_FAIL_BAD_HMAC;
417 #define GET_RING_OBJ(r, pos, f) \
420 vnet_crypto_async_frame_t **ring = (void *) &r[1]; \
421 f = ring[(r->cons.head + pos) & r->mask]; \
425 static_always_inline vnet_crypto_async_frame_t *
426 cryptodev_raw_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
427 u32 *enqueue_thread_idx)
429 cryptodev_main_t *cmt = &cryptodev_main;
430 vnet_crypto_main_t *cm = &crypto_main;
431 cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
432 vnet_crypto_async_frame_t *frame, *frame_ret = 0;
433 u32 n_deq, n_success;
434 u32 n_cached_frame = rte_ring_count (cet->cached_frame), n_room_left;
435 u8 no_job_to_deq = 0;
436 u16 inflight = cet->inflight;
439 n_room_left = CRYPTODEV_DEQ_CACHE_SZ - n_cached_frame - 1;
444 for (i = 0; i < n_cached_frame; i++)
446 vnet_crypto_async_frame_t *f;
448 enum rte_crypto_op_status op_status;
451 GET_RING_OBJ (cet->cached_frame, i, f);
453 if (i < n_cached_frame - 2)
455 vnet_crypto_async_frame_t *f1, *f2;
456 GET_RING_OBJ (cet->cached_frame, i + 1, f1);
457 GET_RING_OBJ (cet->cached_frame, i + 2, f2);
458 clib_prefetch_load (f1);
459 clib_prefetch_load (f2);
462 n_left = f->state & 0x7f;
463 err = f->state & 0x80;
465 for (j = f->n_elts - n_left; j < f->n_elts && inflight; j++)
468 f_ret = rte_cryptodev_raw_dequeue (cet->ctx, &ret, &op_status);
475 case RTE_CRYPTO_OP_STATUS_SUCCESS:
476 f->elts[j].status = VNET_CRYPTO_OP_STATUS_COMPLETED;
479 f->elts[j].status = VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR;
491 f->state = err ? VNET_CRYPTO_FRAME_STATE_ELT_ERROR :
492 VNET_CRYPTO_FRAME_STATE_SUCCESS;
496 f->state = f->n_elts - j;
503 /* to here f is not completed dequeued and no more job can be
506 f->state = f->n_elts - j;
514 rte_ring_sc_dequeue (cet->cached_frame, (void **) &frame_ret);
520 vlib_node_set_interrupt_pending (vlib_get_main_by_index (vm->thread_index),
521 cm->crypto_node_index);
523 /* no point to dequeue further */
524 if (!inflight || no_job_to_deq || !n_room_left)
527 #if RTE_VERSION >= RTE_VERSION_NUM(21, 5, 0, 0)
528 n_deq = rte_cryptodev_raw_dequeue_burst (
529 cet->ctx, cryptodev_get_frame_n_elts, 0, cryptodev_post_dequeue,
530 (void **) &frame, 0, &n_success, &dequeue_status);
532 n_deq = rte_cryptodev_raw_dequeue_burst (
533 cet->ctx, cryptodev_get_frame_n_elts, cryptodev_post_dequeue,
534 (void **) &frame, 0, &n_success, &dequeue_status);
541 no_job_to_deq = n_deq < frame->n_elts;
542 /* we have to cache the frame */
543 if (frame_ret || n_cached_frame || no_job_to_deq)
545 frame->state = frame->n_elts - n_deq;
546 frame->state |= ((n_success < n_deq) << 7);
547 rte_ring_sp_enqueue (cet->cached_frame, (void *) frame);
552 frame->state = n_success == frame->n_elts ?
553 VNET_CRYPTO_FRAME_STATE_SUCCESS :
554 VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
558 /* see if we can dequeue more */
559 while (inflight && n_room_left && !no_job_to_deq)
561 #if RTE_VERSION >= RTE_VERSION_NUM(21, 5, 0, 0)
562 n_deq = rte_cryptodev_raw_dequeue_burst (
563 cet->ctx, cryptodev_get_frame_n_elts, 0, cryptodev_post_dequeue,
564 (void **) &frame, 0, &n_success, &dequeue_status);
566 n_deq = rte_cryptodev_raw_dequeue_burst (
567 cet->ctx, cryptodev_get_frame_n_elts, cryptodev_post_dequeue,
568 (void **) &frame, 0, &n_success, &dequeue_status);
573 no_job_to_deq = n_deq < frame->n_elts;
574 frame->state = frame->n_elts - n_deq;
575 frame->state |= ((n_success < n_deq) << 7);
576 rte_ring_sp_enqueue (cet->cached_frame, (void *) frame);
581 if (inflight < cet->inflight)
584 rte_cryptodev_raw_dequeue_done (cet->ctx, cet->inflight - inflight);
586 cet->inflight = inflight;
591 *nb_elts_processed = frame_ret->n_elts;
592 *enqueue_thread_idx = frame_ret->enqueue_thread_index;
598 static_always_inline int
599 cryptodev_raw_enq_aead_aad_0_enc (vlib_main_t *vm,
600 vnet_crypto_async_frame_t *frame)
602 return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_ENCRYPT, 0);
605 static_always_inline int
606 cryptodev_raw_enq_aead_aad_8_enc (vlib_main_t *vm,
607 vnet_crypto_async_frame_t *frame)
609 return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_ENCRYPT, 8);
611 static_always_inline int
612 cryptodev_raw_enq_aead_aad_12_enc (vlib_main_t *vm,
613 vnet_crypto_async_frame_t *frame)
615 return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_ENCRYPT, 12);
618 static_always_inline int
619 cryptodev_raw_enq_aead_aad_0_dec (vlib_main_t *vm,
620 vnet_crypto_async_frame_t *frame)
622 return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_DECRYPT, 0);
625 static_always_inline int
626 cryptodev_raw_enq_aead_aad_8_dec (vlib_main_t *vm,
627 vnet_crypto_async_frame_t *frame)
629 return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_DECRYPT, 8);
631 static_always_inline int
632 cryptodev_raw_enq_aead_aad_12_dec (vlib_main_t *vm,
633 vnet_crypto_async_frame_t *frame)
635 return cryptodev_raw_aead_enqueue (vm, frame, CRYPTODEV_OP_TYPE_DECRYPT, 12);
638 static_always_inline int
639 cryptodev_raw_enq_linked_alg_enc (vlib_main_t *vm,
640 vnet_crypto_async_frame_t *frame)
642 return cryptodev_frame_linked_algs_enqueue (vm, frame,
643 CRYPTODEV_OP_TYPE_ENCRYPT);
646 static_always_inline int
647 cryptodev_raw_enq_linked_alg_dec (vlib_main_t *vm,
648 vnet_crypto_async_frame_t *frame)
650 return cryptodev_frame_linked_algs_enqueue (vm, frame,
651 CRYPTODEV_OP_TYPE_DECRYPT);
655 cryptodev_register_raw_hdl (vlib_main_t *vm, u32 eidx)
657 cryptodev_main_t *cmt = &cryptodev_main;
658 cryptodev_engine_thread_t *cet;
659 cryptodev_inst_t *cinst;
660 struct rte_cryptodev_info info;
661 struct rte_cryptodev_sym_capability_idx cap_auth_idx;
662 struct rte_cryptodev_sym_capability_idx cap_cipher_idx;
663 struct rte_cryptodev_sym_capability_idx cap_aead_idx;
664 u32 support_raw_api = 1, max_ctx_size = 0;
665 clib_error_t *error = 0;
668 vec_foreach (cinst, cmt->cryptodev_inst)
671 rte_cryptodev_info_get (cinst->dev_id, &info);
672 if (!(info.feature_flags & RTE_CRYPTODEV_FF_SYM_RAW_DP))
678 ctx_size = rte_cryptodev_get_raw_dp_ctx_size (cinst->dev_id);
679 max_ctx_size = clib_max (ctx_size, max_ctx_size);
682 if (!support_raw_api)
683 return cryptodev_register_cop_hdl (vm, eidx);
685 vec_foreach (cet, cmt->per_thread_data)
687 u32 thread_id = cet - cmt->per_thread_data;
688 u32 numa = vlib_get_main_by_index (thread_id)->numa_node;
689 u8 *name = format (0, "cache_frame_ring_%u_%u", numa, thread_id);
692 rte_ring_create ((char *) name, CRYPTODEV_DEQ_CACHE_SZ, numa,
693 RING_F_SC_DEQ | RING_F_SP_ENQ);
695 cet->aad_buf = rte_zmalloc_socket (
696 0, CRYPTODEV_NB_CRYPTO_OPS * CRYPTODEV_MAX_AAD_SIZE,
697 CLIB_CACHE_LINE_BYTES, numa);
698 if (cet->aad_buf == 0)
700 error = clib_error_return (0, "Failed to alloc aad buf");
703 cet->aad_phy_addr = rte_malloc_virt2iova (cet->aad_buf);
706 rte_zmalloc_socket (0, max_ctx_size, CLIB_CACHE_LINE_BYTES, numa);
709 error = clib_error_return (0, "Failed to alloc raw dp ctx");
713 if (cet->cached_frame == 0)
715 error = clib_error_return (0, "Failed to alloc frame ring %s", name);
722 #define _(a, b, c, d, e, f, g) \
723 cap_aead_idx.type = RTE_CRYPTO_SYM_XFORM_AEAD; \
724 cap_aead_idx.algo.aead = RTE_CRYPTO_##b##_##c; \
725 if (cryptodev_check_cap_support (&cap_aead_idx, g, e, f)) \
727 vnet_crypto_register_enqueue_handler ( \
728 vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_ENC, \
729 cryptodev_raw_enq_aead_aad_##f##_enc); \
730 vnet_crypto_register_enqueue_handler ( \
731 vm, eidx, VNET_CRYPTO_OP_##a##_TAG##e##_AAD##f##_DEC, \
732 cryptodev_raw_enq_aead_aad_##f##_dec); \
735 foreach_vnet_aead_crypto_conversion
738 #define _(a, b, c, d, e) \
739 cap_auth_idx.type = RTE_CRYPTO_SYM_XFORM_AUTH; \
740 cap_auth_idx.algo.auth = RTE_CRYPTO_AUTH_##d##_HMAC; \
741 cap_cipher_idx.type = RTE_CRYPTO_SYM_XFORM_CIPHER; \
742 cap_cipher_idx.algo.cipher = RTE_CRYPTO_CIPHER_##b; \
743 if (cryptodev_check_cap_support (&cap_cipher_idx, c, -1, -1) && \
744 cryptodev_check_cap_support (&cap_auth_idx, -1, e, -1)) \
746 vnet_crypto_register_enqueue_handler ( \
747 vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_ENC, \
748 cryptodev_raw_enq_linked_alg_enc); \
749 vnet_crypto_register_enqueue_handler ( \
750 vm, eidx, VNET_CRYPTO_OP_##a##_##d##_TAG##e##_DEC, \
751 cryptodev_raw_enq_linked_alg_dec); \
754 foreach_cryptodev_link_async_alg
758 vnet_crypto_register_dequeue_handler (vm, eidx, cryptodev_raw_dequeue);
765 vec_foreach (cet, cmt->per_thread_data)
767 if (cet->cached_frame)
768 rte_ring_free (cet->cached_frame);