};
static clib_error_t *
-cryptodev_show_sw_rings_fn (vlib_main_t *vm, unformat_input_t *input,
- vlib_cli_command_t *cmd)
+cryptodev_show_cache_rings_fn (vlib_main_t *vm, unformat_input_t *input,
+ vlib_cli_command_t *cmd)
{
cryptodev_main_t *cmt = &cryptodev_main;
u32 thread_index = 0;
vec_foreach_index (thread_index, cmt->per_thread_data)
{
cryptodev_engine_thread_t *cet = cmt->per_thread_data + thread_index;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ u16 head = ring->head;
+ u16 tail = ring->tail;
+ u16 n_cached = ((head == tail) && (ring->frames[head].f == 0)) ?
+ 0 :
+ ((head == tail) && (ring->frames[head].f != 0)) ?
+ (CRYPTODEV_CACHE_QUEUE_MASK + 1) :
+ (head > tail) ?
+ (head - tail) :
+ (CRYPTODEV_CACHE_QUEUE_MASK - tail + head);
+
+ u16 enq_head = ring->enq_head;
+ u16 deq_tail = ring->deq_tail;
+ u16 n_frames_inflight =
+ ((enq_head == deq_tail) && (ring->frames[enq_head].f == 0)) ?
+ 0 :
+ ((enq_head == deq_tail) && (ring->frames[enq_head].f != 0)) ?
+ CRYPTODEV_CACHE_QUEUE_MASK + 1 :
+ (enq_head > deq_tail) ?
+ (enq_head - deq_tail) :
+ (CRYPTODEV_CACHE_QUEUE_MASK - deq_tail + enq_head);
+
+ u16 n_frames_processed =
+ ((tail == deq_tail) && (ring->frames[deq_tail].f == 0)) ?
+ 0 :
+ ((tail == deq_tail) && (ring->frames[deq_tail].f != 0)) ?
+ (CRYPTODEV_CACHE_QUEUE_MASK + 1) :
+ (deq_tail > tail) ? (deq_tail - tail) :
+ (CRYPTODEV_CACHE_QUEUE_MASK - tail + deq_tail);
+
if (vlib_num_workers () > 0 && thread_index == 0)
continue;
vlib_cli_output (vm, "\n\n");
- vlib_cli_output (vm, "Frames total: %d", cet->frames_on_ring);
- vlib_cli_output (vm, "Frames pending in a ring: %d",
- cet->frames_on_ring - cet->enqueued_not_dequeueq -
- cet->deqeued_not_returned);
+ vlib_cli_output (vm, "Frames total: %d", n_cached);
+ vlib_cli_output (vm, "Frames pending in the ring: %d",
+ n_cached - n_frames_inflight - n_frames_processed);
vlib_cli_output (vm, "Frames enqueued but not dequeued: %d",
- cet->enqueued_not_dequeueq);
+ n_frames_inflight);
vlib_cli_output (vm, "Frames dequed but not returned: %d",
- cet->deqeued_not_returned);
+ n_frames_processed);
vlib_cli_output (vm, "inflight: %d", cet->inflight);
- vlib_cli_output (vm, "Head: %d", cet->frame_ring.head);
- vlib_cli_output (vm, "Tail: %d", cet->frame_ring.tail);
+ vlib_cli_output (vm, "Head: %d", ring->head);
+ vlib_cli_output (vm, "Tail: %d", ring->tail);
vlib_cli_output (vm, "\n\n");
}
return 0;
}
VLIB_CLI_COMMAND (show_cryptodev_sw_rings, static) = {
- .path = "show cryptodev sw-ring status",
- .short_help = "show status of all cryptodev software rings",
- .function = cryptodev_show_sw_rings_fn,
+ .path = "show cryptodev cache status",
+ .short_help = "show status of all cryptodev cache rings",
+ .function = cryptodev_show_cache_rings_fn,
};
static clib_error_t *
typedef struct
{
vnet_crypto_async_frame_t *f;
+ union
+ {
+ struct
+ {
+ /* index of frame elt where enque to
+ * the crypto engine is happening */
+ u8 enq_elts_head;
+ /* index of the frame elt where dequeue
+ * from the crypto engine is happening */
+ u8 deq_elts_tail;
+ u8 elts_inflight;
- u8 enqueued;
- u8 dequeued;
- u8 deq_state;
- u8 frame_inflight;
+ u8 op_type;
+ u8 aad_len;
+ u8 n_elts;
+ u16 reserved;
+ };
+ u64 raw;
+ };
- u8 op_type;
- u8 aad_len;
- u8 n_elts;
- u8 reserved;
-} cryptodev_async_ring_elt;
+ u64 frame_elts_errs_mask;
+} cryptodev_cache_ring_elt_t;
typedef struct
{
- cryptodev_async_ring_elt frames[VNET_CRYPTO_FRAME_POOL_SIZE];
- uint16_t head;
- uint16_t tail;
- uint16_t enq; /*record the frame currently being enqueued */
- uint16_t deq; /*record the frame currently being dequeued */
-} cryptodev_async_frame_sw_ring;
+ cryptodev_cache_ring_elt_t frames[VNET_CRYPTO_FRAME_POOL_SIZE];
+
+ union
+ {
+ struct
+ {
+ /* head of the cache ring */
+ u16 head;
+ /* tail of the cache ring */
+ u16 tail;
+ /* index of the frame where enqueue
+ * to the crypto engine is happening */
+ u16 enq_head;
+ /* index of the frame where dequeue
+ * from the crypto engine is happening */
+ u16 deq_tail;
+ };
+ u64 raw;
+ };
+} cryptodev_cache_ring_t;
typedef struct
{
};
};
- cryptodev_async_frame_sw_ring frame_ring;
+ cryptodev_cache_ring_t cache_ring;
u16 cryptodev_id;
u16 cryptodev_q;
- u16 frames_on_ring;
- u16 enqueued_not_dequeueq;
- u16 deqeued_not_returned;
- u16 pending_to_qat;
u16 inflight;
} cryptodev_engine_thread_t;
extern cryptodev_main_t cryptodev_main;
+#define CRYPTODEV_CACHE_RING_GET_FRAME(r, i) \
+ ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].f)
+
+#define CRYPTODEV_CACHE_RING_GET_ERR_MASK(r, i) \
+ ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].frame_elts_errs_mask)
+
+#define CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT(r, i) \
+ (((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].enq_elts_head) - \
+ ((r)->frames[(i) &CRYPTODEV_CACHE_QUEUE_MASK].deq_elts_tail))
+
static_always_inline void
-cryptodev_mark_frame_err_status (vnet_crypto_async_frame_t *f,
- vnet_crypto_op_status_t s,
- vnet_crypto_async_frame_state_t fs)
+cryptodev_cache_ring_update_enq_head (cryptodev_cache_ring_t *r,
+ vnet_crypto_async_frame_t *f)
+{
+ if (r->frames[r->enq_head].enq_elts_head == f->n_elts)
+ {
+ r->enq_head++;
+ r->enq_head &= CRYPTODEV_CACHE_QUEUE_MASK;
+ f->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
+ }
+}
+
+static_always_inline bool
+cryptodev_cache_ring_update_deq_tail (cryptodev_cache_ring_t *r,
+ u16 *const deq)
{
- u32 n_elts = f->n_elts, i;
+ if (r->frames[*deq].deq_elts_tail == r->frames[*deq].n_elts)
+ {
+ *deq += 1;
+ *deq &= CRYPTODEV_CACHE_QUEUE_MASK;
+ return 1;
+ }
+
+ return 0;
+}
+static_always_inline u64
+cryptodev_mark_frame_fill_err (vnet_crypto_async_frame_t *f, u64 current_err,
+ u16 index, u16 n, vnet_crypto_op_status_t op_s)
+{
+ u64 err = current_err;
+ u16 i;
+
+ ERROR_ASSERT (index + n <= VNET_CRYPTO_FRAME_SIZE);
+ ERROR_ASSERT (op_s != VNET_CRYPTO_OP_STATUS_COMPLETED);
+
+ for (i = index; i < (index + n); i++)
+ f->elts[i].status = op_s;
+
+ err |= (~(~(0u) << n) << index);
+
+ return err;
+}
+
+static_always_inline cryptodev_cache_ring_elt_t *
+cryptodev_cache_ring_push (cryptodev_cache_ring_t *r,
+ vnet_crypto_async_frame_t *f)
+{
+ u16 head = r->head;
+ cryptodev_cache_ring_elt_t *ring_elt = &r->frames[head];
+ /**
+ * in debug mode we do the ring sanity test when a frame is enqueued to
+ * the ring.
+ **/
+#if CLIB_DEBUG > 0
+ u16 tail = r->tail;
+ u16 n_cached = (head >= tail) ? (head - tail) :
+ (CRYPTODEV_CACHE_QUEUE_MASK - tail + head);
+ ERROR_ASSERT (n_cached < VNET_CRYPTO_FRAME_POOL_SIZE);
+ ERROR_ASSERT (r->raw == 0 && r->frames[head].raw == 0 &&
+ r->frames[head].f == 0);
+#endif
+ ring_elt->f = f;
+ ring_elt->n_elts = f->n_elts;
+ /* update head */
+ r->head++;
+ r->head &= CRYPTODEV_CACHE_QUEUE_MASK;
+ return ring_elt;
+}
+
+static_always_inline vnet_crypto_async_frame_t *
+cryptodev_cache_ring_pop (cryptodev_cache_ring_t *r)
+{
+ vnet_crypto_async_frame_t *f;
+ u16 tail = r->tail;
+ cryptodev_cache_ring_elt_t *ring_elt = &r->frames[tail];
+
+ ERROR_ASSERT (r->frames[r->head].raw == 0 ? r->head != tail : 1);
+ ERROR_ASSERT (r->frames[tail].raw != 0);
+ ERROR_ASSERT (ring_elt->deq_elts_tail == ring_elt->enq_elts_head &&
+ ring_elt->deq_elts_tail == ring_elt->n_elts);
+
+ f = CRYPTODEV_CACHE_RING_GET_FRAME (r, tail);
+ f->state = CRYPTODEV_CACHE_RING_GET_ERR_MASK (r, r->tail) == 0 ?
+ VNET_CRYPTO_FRAME_STATE_SUCCESS :
+ VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
+
+ clib_memset (ring_elt, 0, sizeof (*ring_elt));
+ r->tail++;
+ r->tail &= CRYPTODEV_CACHE_QUEUE_MASK;
- for (i = 0; i < n_elts; i++)
- f->elts[i].status = s;
- f->state = fs;
+ return f;
}
int cryptodev_session_create (vlib_main_t *vm, vnet_crypto_key_index_t idx,
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
- cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head];
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ ERROR_ASSERT (frame != 0);
+ ERROR_ASSERT (frame->n_elts > 0);
+ cryptodev_cache_ring_elt_t *ring_elt =
+ cryptodev_cache_ring_push (ring, frame);
- cet->frames_on_ring++;
- ring_elt->f = frame;
- ring_elt->n_elts = frame->n_elts;
ring_elt->aad_len = 1;
ring_elt->op_type = (u8) op_type;
- ring->head++;
- ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
return 0;
}
cryptodev_main_t *cmt = &cryptodev_main;
clib_pmalloc_main_t *pm = vm->physmem_main.pmalloc_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ u16 *const enq = &ring->enq_head;
vnet_crypto_async_frame_elt_t *fe;
cryptodev_session_t *sess = 0;
cryptodev_op_t *cops[CRYPTODE_ENQ_MAX] = {};
return;
max_to_enq = clib_min (CRYPTODE_ENQ_MAX,
- frame->n_elts - ring->frames[ring->enq].enqueued);
+ frame->n_elts - ring->frames[*enq].enq_elts_head);
if (cet->inflight + max_to_enq > CRYPTODEV_MAX_INFLIGHT)
return;
if (PREDICT_FALSE (
rte_mempool_get_bulk (cet->cop_pool, (void **) cops, n_elts) < 0))
{
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED);
+ cryptodev_mark_frame_fill_err (
+ frame, ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ ring->frames[*enq].deq_elts_tail += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
return;
}
- fe = frame->elts + ring->frames[ring->enq].enqueued;
- bi = frame->buffer_indices + ring->frames[ring->enq].enqueued;
+ fe = frame->elts + ring->frames[*enq].enq_elts_head;
+ bi = frame->buffer_indices + ring->frames[*enq].enq_elts_head;
while (n_elts)
{
if (PREDICT_FALSE (
cryptodev_session_create (vm, last_key_index, 0) < 0))
{
- cryptodev_mark_frame_err_status (
- frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED);
+ cryptodev_mark_frame_fill_err (
+ frame, ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
goto error_exit;
}
}
cryptodev_validate_mbuf (sop->m_src, b);
clib_memcpy_fast (cop[0]->iv, fe->iv, 16);
- ring->frames[ring->enq].enqueued++;
+ ring->frames[*enq].enq_elts_head++;
cop++;
bi++;
fe++;
n_enqueue =
rte_cryptodev_enqueue_burst (cet->cryptodev_id, cet->cryptodev_q,
(struct rte_crypto_op **) cops, max_to_enq);
- ASSERT (n_enqueue == max_to_enq);
+ ERROR_ASSERT (n_enqueue == max_to_enq);
cet->inflight += max_to_enq;
- ring->frames[ring->enq].frame_inflight += max_to_enq;
- if (ring->frames[ring->enq].enqueued == frame->n_elts)
- {
- cet->frame_ring.enq++;
- cet->frame_ring.enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
- frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
- }
-
+ cryptodev_cache_ring_update_enq_head (ring, frame);
return;
error_exit:
- ring->enq++;
- ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ ring->frames[*enq].deq_elts_tail += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
rte_mempool_put_bulk (cet->cop_pool, (void **) cops, max_to_enq);
}
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
- cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head];
- cet->frames_on_ring++;
- ring_elt->f = frame;
- ring_elt->n_elts = frame->n_elts;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ ERROR_ASSERT (frame != 0);
+ ERROR_ASSERT (frame->n_elts > 0);
+ cryptodev_cache_ring_elt_t *ring_elt =
+ cryptodev_cache_ring_push (ring, frame);
ring_elt->aad_len = aad_len;
ring_elt->op_type = (u8) op_type;
- ring->head++;
- ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
-
return 0;
}
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ u16 *const enq = &ring->enq_head;
clib_pmalloc_main_t *pm = vm->physmem_main.pmalloc_main;
vnet_crypto_async_frame_elt_t *fe;
cryptodev_session_t *sess = 0;
u32 *bi = 0;
u32 n_enqueue = 0, n_elts;
u32 last_key_index = ~0;
- u16 left_to_enq = frame->n_elts - ring->frames[ring->enq].enqueued;
+ u16 left_to_enq = frame->n_elts - ring->frames[*enq].enq_elts_head;
const u16 max_to_enq = clib_min (CRYPTODE_ENQ_MAX, left_to_enq);
if (PREDICT_FALSE (frame == 0 || frame->n_elts == 0))
if (PREDICT_FALSE (
rte_mempool_get_bulk (cet->cop_pool, (void **) cops, n_elts) < 0))
{
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED);
+ cryptodev_mark_frame_fill_err (
+ frame, ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ ring->frames[*enq].deq_elts_tail += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
return -1;
}
- fe = frame->elts + ring->frames[ring->enq].enqueued;
- bi = frame->buffer_indices + ring->frames[ring->enq].enqueued;
+ fe = frame->elts + ring->frames[*enq].enq_elts_head;
+ bi = frame->buffer_indices + ring->frames[*enq].enq_elts_head;
while (n_elts)
{
if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index,
aad_len) < 0))
{
- cryptodev_mark_frame_err_status (
- frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED);
+ cryptodev_mark_frame_fill_err (
+ frame, ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
goto error_exit;
}
}
if (PREDICT_FALSE (cryptodev_session_create (vm, last_key_index,
aad_len) < 0))
{
- cryptodev_mark_frame_err_status (
- frame, VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED);
+ cryptodev_mark_frame_fill_err (
+ frame, ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
goto error_exit;
}
}
clib_memcpy_fast (cop[0]->iv, fe->iv, 12);
clib_memcpy_fast (cop[0]->aad, fe->aad, aad_len);
- ring->frames[ring->enq].enqueued++;
cop++;
bi++;
fe++;
n_enqueue =
rte_cryptodev_enqueue_burst (cet->cryptodev_id, cet->cryptodev_q,
(struct rte_crypto_op **) cops, max_to_enq);
- ASSERT (n_enqueue == max_to_enq);
+ ERROR_ASSERT (n_enqueue == max_to_enq);
cet->inflight += max_to_enq;
- ring->frames[ring->enq].frame_inflight += max_to_enq;
- if (ring->frames[ring->enq].enqueued == frame->n_elts)
- {
- ring->enq++;
- ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
- frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
- cet->enqueued_not_dequeueq++;
- }
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
return 0;
error_exit:
- ring->enq++;
- ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ ring->frames[*enq].deq_elts_tail += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
rte_mempool_put_bulk (cet->cop_pool, (void **) cops, max_to_enq);
return -1;
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
vnet_crypto_async_frame_t *frame = NULL;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ u16 *const deq = &ring->deq_tail;
u16 n_deq, idx, left_to_deq, i;
u16 max_to_deq = 0;
u16 inflight = cet->inflight;
cryptodev_op_t *cops[CRYPTODE_DEQ_MAX] = {};
cryptodev_op_t **cop = cops;
vnet_crypto_async_frame_elt_t *fe;
- u32 n_elts;
- u32 ss0 = 0, ss1 = 0, ss2 = 0, ss3 = 0; /* sum of status */
+ u32 n_elts, n;
+ u64 err0 = 0, err1 = 0, err2 = 0, err3 = 0; /* partial errors mask */
- idx = ring->deq;
+ idx = ring->deq_tail;
for (i = 0; i < VNET_CRYPTO_FRAME_POOL_SIZE; i++)
{
- if (PREDICT_TRUE (ring->frames[idx].frame_inflight > 0))
+ u32 frame_inflight =
+ CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT (ring, idx);
+
+ if (PREDICT_TRUE (frame_inflight > 0))
break;
idx++;
idx &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
}
- ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE);
- ring->deq = idx;
+ ERROR_ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE);
+ ring->deq_tail = idx;
left_to_deq =
- ring->frames[ring->deq].f->n_elts - ring->frames[ring->deq].dequeued;
+ ring->frames[*deq].f->n_elts - ring->frames[*deq].deq_elts_tail;
max_to_deq = clib_min (left_to_deq, CRYPTODE_DEQ_MAX);
/* deq field can be used to track frame that is currently dequeued
if (n_deq == 0)
return dequeue_more;
- ss0 = ring->frames[ring->deq].deq_state;
- ss1 = ring->frames[ring->deq].deq_state;
- ss2 = ring->frames[ring->deq].deq_state;
- ss3 = ring->frames[ring->deq].deq_state;
-
- frame = ring->frames[ring->deq].f;
- fe = frame->elts + ring->frames[ring->deq].dequeued;
+ frame = ring->frames[*deq].f;
+ fe = frame->elts + ring->frames[*deq].deq_elts_tail;
n_elts = n_deq;
+ n = ring->frames[*deq].deq_elts_tail;
+
while (n_elts > 4)
{
- ss0 |= fe[0].status = cryptodev_status_conversion[cop[0]->op.status];
- ss1 |= fe[1].status = cryptodev_status_conversion[cop[1]->op.status];
- ss2 |= fe[2].status = cryptodev_status_conversion[cop[2]->op.status];
- ss3 |= fe[3].status = cryptodev_status_conversion[cop[3]->op.status];
+ fe[0].status = cryptodev_status_conversion[cop[0]->op.status];
+ fe[1].status = cryptodev_status_conversion[cop[1]->op.status];
+ fe[2].status = cryptodev_status_conversion[cop[2]->op.status];
+ fe[3].status = cryptodev_status_conversion[cop[3]->op.status];
+
+ err0 |= (fe[0].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << n;
+ err1 |= (fe[1].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << (n + 1);
+ err2 |= (fe[2].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << (n + 2);
+ err3 |= (fe[3].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << (n + 3);
cop += 4;
fe += 4;
n_elts -= 4;
+ n += 4;
}
while (n_elts)
{
- ss0 |= fe[0].status = cryptodev_status_conversion[cop[0]->op.status];
+ fe[0].status = cryptodev_status_conversion[cop[0]->op.status];
+ err0 |= (fe[0].status == VNET_CRYPTO_OP_STATUS_COMPLETED) << n;
+ n++;
fe++;
cop++;
n_elts--;
}
- ring->frames[ring->deq].deq_state |= (u8) (ss0 | ss1 | ss2 | ss3);
+ ring->frames[*deq].frame_elts_errs_mask |= (err0 | err1 | err2 | err3);
rte_mempool_put_bulk (cet->cop_pool, (void **) cops, n_deq);
inflight -= n_deq;
- ring->frames[ring->deq].dequeued += n_deq;
- ring->frames[ring->deq].frame_inflight -= n_deq;
- if (ring->frames[ring->deq].dequeued == ring->frames[ring->deq].n_elts)
+ ring->frames[*deq].deq_elts_tail += n_deq;
+ if (cryptodev_cache_ring_update_deq_tail (ring, deq))
{
- frame->state =
- (ss0 | ss1 | ss2 | ss3) == VNET_CRYPTO_OP_STATUS_COMPLETED ?
- VNET_CRYPTO_FRAME_STATE_SUCCESS :
- VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
-
*nb_elts_processed = frame->n_elts;
*enqueue_thread_idx = frame->enqueue_thread_index;
- cet->deqeued_not_returned++;
- cet->enqueued_not_dequeueq--;
- ring->deq++;
- ring->deq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
dequeue_more = (max_to_deq < CRYPTODE_DEQ_MAX);
}
}
static_always_inline void
-cryptodev_enqueue_frame (vlib_main_t *vm, cryptodev_async_ring_elt *ring_elt)
+cryptodev_enqueue_frame (vlib_main_t *vm, cryptodev_cache_ring_elt_t *ring_elt)
{
cryptodev_op_type_t op_type = (cryptodev_op_type_t) ring_elt->op_type;
u8 linked_or_aad_len = ring_elt->aad_len;
cryptodev_main_t *cmt = &cryptodev_main;
vnet_crypto_main_t *cm = &crypto_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ cryptodev_cache_ring_elt_t *ring_elt = &ring->frames[ring->tail];
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
- cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->tail];
vnet_crypto_async_frame_t *ret_frame = 0;
u8 dequeue_more = 1;
enqueue_thread_idx);
}
- if (PREDICT_TRUE (ring->frames[ring->enq].f != 0))
- cryptodev_enqueue_frame (vm, &ring->frames[ring->enq]);
+ if (PREDICT_TRUE (ring->frames[ring->enq_head].f != 0))
+ cryptodev_enqueue_frame (vm, &ring->frames[ring->enq_head]);
if (PREDICT_TRUE (ring_elt->f != 0))
{
- if ((ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_SUCCESS ||
- ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_ELT_ERROR) &&
- ring_elt->enqueued == ring_elt->dequeued)
+ if (ring_elt->enq_elts_head == ring_elt->deq_elts_tail)
{
vlib_node_set_interrupt_pending (
vlib_get_main_by_index (vm->thread_index), cm->crypto_node_index);
- ret_frame = ring_elt->f;
- memset (ring_elt, 0, sizeof (*ring_elt));
- ring->tail += 1;
- ring->tail &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
- cet->frames_on_ring--;
- cet->deqeued_not_returned--;
+ ret_frame = cryptodev_cache_ring_pop (ring);
return ret_frame;
}
}
{
union rte_cryptodev_session_ctx sess_ctx;
- ASSERT (cet->reset_sess != 0);
+ ERROR_ASSERT (cet->reset_sess != 0);
sess_ctx.crypto_sess = cet->reset_sess;
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
- cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head];
- cet->frames_on_ring++;
- ring_elt->f = frame;
- ring_elt->n_elts = frame->n_elts;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ ERROR_ASSERT (frame != 0);
+ ERROR_ASSERT (frame->n_elts > 0);
+ cryptodev_cache_ring_elt_t *ring_elt =
+ cryptodev_cache_ring_push (ring, frame);
+
ring_elt->aad_len = 1;
ring_elt->op_type = (u8) op_type;
- ring->head++;
- ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
return 0;
}
vlib_buffer_t **b;
struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL];
struct rte_crypto_va_iova_ptr iv_vec, digest_vec;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ u16 *const enq = &ring->enq_head;
u32 n_elts;
u32 last_key_index = ~0;
i16 min_ofs;
u32 max_end;
u32 max_to_enq = clib_min (CRYPTODE_ENQ_MAX,
- frame->n_elts - ring->frames[ring->enq].enqueued);
+ frame->n_elts - ring->frames[*enq].enq_elts_head);
u8 is_update = 0;
int status;
vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts);
- b = cet->b + ring->frames[ring->enq].enqueued;
- fe = frame->elts + ring->frames[ring->enq].enqueued;
+ b = cet->b + ring->frames[*enq].enq_elts_head;
+ fe = frame->elts + ring->frames[*enq].enq_elts_head;
while (n_elts)
{
if (PREDICT_FALSE (status < 0))
goto error_exit;
- ring->frames[ring->enq].enqueued += 1;
+ ring->frames[*enq].enq_elts_head += 1;
b++;
fe++;
n_elts--;
goto error_exit;
cet->inflight += max_to_enq;
- ring->frames[ring->enq].frame_inflight += max_to_enq;
- if (ring->frames[ring->enq].enqueued == frame->n_elts)
- {
- cet->frame_ring.enq += 1;
- cet->frame_ring.enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
- frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
- }
+ cryptodev_cache_ring_update_enq_head (ring, frame);
return;
error_exit:
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_ELT_ERROR);
+ cryptodev_mark_frame_fill_err (frame,
+ ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ ring->frames[*enq].deq_elts_tail += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
cryptodev_reset_ctx (cet);
- cet->frame_ring.enq += 1;
- cet->frame_ring.enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
+
return;
}
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
- cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->head];
- cet->frames_on_ring++;
- ring_elt->f = frame;
- ring_elt->n_elts = frame->n_elts;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ ERROR_ASSERT (frame != 0);
+ ERROR_ASSERT (frame->n_elts > 0);
+ cryptodev_cache_ring_elt_t *ring_elt =
+ cryptodev_cache_ring_push (ring, frame);
+
ring_elt->aad_len = aad_len;
ring_elt->op_type = (u8) op_type;
- ring->head++;
- ring->head &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
return 0;
}
{
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
vnet_crypto_async_frame_elt_t *fe;
vlib_buffer_t **b;
u32 n_elts;
struct rte_crypto_vec vec[CRYPTODEV_MAX_N_SGL];
struct rte_crypto_va_iova_ptr iv_vec, digest_vec, aad_vec;
u32 last_key_index = ~0;
- u16 left_to_enq = frame->n_elts - ring->frames[ring->enq].enqueued;
+ u16 *const enq = &ring->enq_head;
+ u16 left_to_enq = frame->n_elts - ring->frames[*enq].enq_elts_head;
u16 max_to_enq = clib_min (CRYPTODE_ENQ_MAX, left_to_enq);
u8 is_update = 0;
int status;
vlib_get_buffers (vm, frame->buffer_indices, cet->b, frame->n_elts);
- fe = frame->elts + ring->frames[ring->enq].enqueued;
- b = cet->b + ring->frames[ring->enq].enqueued;
+ fe = frame->elts + ring->frames[*enq].enq_elts_head;
+ b = cet->b + ring->frames[*enq].enq_elts_head;
cofs.raw = 0;
while (n_elts)
if (PREDICT_FALSE (status < 0))
goto error_exit;
- ring->frames[ring->enq].enqueued += 1;
+ ring->frames[*enq].enq_elts_head += 1;
fe++;
b++;
n_elts--;
goto error_exit;
cet->inflight += max_to_enq;
- ring->frames[ring->enq].frame_inflight += max_to_enq;
- if (ring->frames[ring->enq].enqueued == frame->n_elts)
- {
- ring->enq += 1;
- ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
- frame->state = VNET_CRYPTO_FRAME_STATE_NOT_PROCESSED;
- cet->enqueued_not_dequeueq++;
- }
-
+ cryptodev_cache_ring_update_enq_head (ring, frame);
return;
error_exit:
- cryptodev_mark_frame_err_status (frame,
- VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR,
- VNET_CRYPTO_FRAME_STATE_ELT_ERROR);
+ cryptodev_mark_frame_fill_err (frame,
+ ring->frames[*enq].frame_elts_errs_mask,
+ ring->frames[*enq].enq_elts_head, max_to_enq,
+ VNET_CRYPTO_OP_STATUS_FAIL_ENGINE_ERR);
+ ring->frames[*enq].enq_elts_head += max_to_enq;
+ ring->frames[*enq].deq_elts_tail += max_to_enq;
+ cryptodev_cache_ring_update_enq_head (ring, frame);
cryptodev_reset_ctx (cet);
- ring->enq += 1;
- ring->enq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
return;
}
cryptodev_main_t *cmt = &cryptodev_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
vnet_crypto_async_frame_t *frame;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ u16 *const deq = &ring->deq_tail;
u32 n_success;
u16 n_deq, indice, i, left_to_deq;
u16 max_to_deq = 0;
u8 dequeue_more = 0;
int dequeue_status;
- indice = ring->deq;
+ indice = *deq;
for (i = 0; i < VNET_CRYPTO_FRAME_POOL_SIZE; i++)
{
- if (PREDICT_TRUE (ring->frames[indice].frame_inflight > 0))
+ if (PREDICT_TRUE (
+ CRYPTODEV_CACHE_RING_GET_FRAME_ELTS_INFLIGHT (ring, indice) > 0))
break;
indice += 1;
- indice &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
+ indice &= CRYPTODEV_CACHE_QUEUE_MASK;
}
- ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE);
+ ERROR_ASSERT (i != VNET_CRYPTO_FRAME_POOL_SIZE);
- ring->deq = indice;
+ *deq = indice;
- left_to_deq =
- ring->frames[ring->deq].f->n_elts - ring->frames[ring->deq].dequeued;
+ left_to_deq = ring->frames[*deq].n_elts - ring->frames[*deq].deq_elts_tail;
max_to_deq = clib_min (left_to_deq, CRYPTODE_DEQ_MAX);
/* you can use deq field to track frame that is currently dequeued */
cet->ctx, NULL, max_to_deq, cryptodev_post_dequeue, (void **) &frame, 0,
&n_success, &dequeue_status);
- if (!n_deq)
+ if (n_deq == 0)
return dequeue_more;
inflight -= n_deq;
- ring->frames[ring->deq].dequeued += n_deq;
- ring->frames[ring->deq].deq_state += n_success;
- ring->frames[ring->deq].frame_inflight -= n_deq;
+ if (PREDICT_FALSE (n_success < n_deq))
+ {
+ u16 idx = ring->frames[*deq].deq_elts_tail;
- if (ring->frames[ring->deq].dequeued == ring->frames[ring->deq].n_elts)
+ for (i = 0; i < n_deq; i++)
+ {
+ if (frame->elts[idx + i].status != VNET_CRYPTO_OP_STATUS_COMPLETED)
+ ring->frames[*deq].frame_elts_errs_mask |= 1 << (idx + i);
+ }
+ }
+ ring->frames[*deq].deq_elts_tail += n_deq;
+
+ if (cryptodev_cache_ring_update_deq_tail (ring, deq))
{
- frame->state = ring->frames[ring->deq].deq_state == frame->n_elts ?
- VNET_CRYPTO_FRAME_STATE_SUCCESS :
- VNET_CRYPTO_FRAME_STATE_ELT_ERROR;
*nb_elts_processed = frame->n_elts;
*enqueue_thread_idx = frame->enqueue_thread_index;
- cet->deqeued_not_returned++;
- cet->enqueued_not_dequeueq--;
- ring->deq += 1;
- ring->deq &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
dequeue_more = max_to_deq < CRYPTODE_DEQ_MAX;
}
int res =
rte_cryptodev_raw_dequeue_done (cet->ctx, cet->inflight - inflight);
- ASSERT (res == 0);
+ ERROR_ASSERT (res == 0);
cet->inflight = inflight;
return dequeue_more;
}
static_always_inline void
-cryptodev_enqueue_frame (vlib_main_t *vm, cryptodev_async_ring_elt *ring_elt)
+cryptodev_enqueue_frame_to_qat (vlib_main_t *vm,
+ cryptodev_cache_ring_elt_t *ring_elt)
{
cryptodev_op_type_t op_type = (cryptodev_op_type_t) ring_elt->op_type;
u8 linked_or_aad_len = ring_elt->aad_len;
cryptodev_main_t *cmt = &cryptodev_main;
vnet_crypto_main_t *cm = &crypto_main;
cryptodev_engine_thread_t *cet = cmt->per_thread_data + vm->thread_index;
- cryptodev_async_frame_sw_ring *ring = &cet->frame_ring;
- cryptodev_async_ring_elt *ring_elt = &ring->frames[ring->tail];
+ cryptodev_cache_ring_t *ring = &cet->cache_ring;
+ cryptodev_cache_ring_elt_t *ring_elt = &ring->frames[ring->tail];
vnet_crypto_async_frame_t *ret_frame = 0;
u8 dequeue_more = 1;
enqueue_thread_idx);
}
- if (PREDICT_TRUE (ring->frames[ring->enq].f != 0))
- cryptodev_enqueue_frame (vm, &ring->frames[ring->enq]);
+ if (PREDICT_TRUE (ring->frames[ring->enq_head].f != 0))
+ cryptodev_enqueue_frame_to_qat (vm, &ring->frames[ring->enq_head]);
if (PREDICT_TRUE (ring_elt->f != 0))
{
- if ((ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_SUCCESS ||
- ring_elt->f->state == VNET_CRYPTO_FRAME_STATE_ELT_ERROR) &&
- ring_elt->enqueued == ring_elt->dequeued)
+ if (ring_elt->enq_elts_head == ring_elt->deq_elts_tail)
{
vlib_node_set_interrupt_pending (
vlib_get_main_by_index (vm->thread_index), cm->crypto_node_index);
- ret_frame = ring_elt->f;
- ring_elt->f = 0;
- ring_elt->dequeued = 0;
- ring_elt->enqueued = 0;
- ring_elt->deq_state = 0;
- ring->tail += 1;
- ring->tail &= (VNET_CRYPTO_FRAME_POOL_SIZE - 1);
- cet->frames_on_ring--;
- cet->deqeued_not_returned--;
+ ret_frame = cryptodev_cache_ring_pop (ring);
+
return ret_frame;
}
}
{
u32 thread_id = cet - cmt->per_thread_data;
u32 numa = vlib_get_main_by_index (thread_id)->numa_node;
- u8 *name = format (0, "cache_frame_ring_%u_%u", numa, thread_id);
+ u8 *name = format (0, "cache_cache_ring_%u_%u", numa, thread_id);
cet->aad_buf = rte_zmalloc_socket (
0, CRYPTODEV_NB_CRYPTO_OPS * CRYPTODEV_MAX_AAD_SIZE,
Code Review / vpp.git / commitdiff