process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
&state);
f->state = state;
- }
+}
+
+static_always_inline void
+crypto_sw_scheduler_process_link (vlib_main_t *vm,
+ crypto_sw_scheduler_main_t *cm,
+ crypto_sw_scheduler_per_thread_data_t *ptd,
+ vnet_crypto_async_frame_t *f, u32 crypto_op,
+ u32 auth_op, u16 digest_len, u8 is_enc)
+{
+ vnet_crypto_async_frame_elt_t *fe;
+ u32 *bi;
+ u32 n_elts = f->n_elts;
+ u8 state = VNET_CRYPTO_FRAME_STATE_SUCCESS;
- static_always_inline void
- crypto_sw_scheduler_process_link (
- vlib_main_t *vm, crypto_sw_scheduler_main_t *cm,
- crypto_sw_scheduler_per_thread_data_t *ptd, vnet_crypto_async_frame_t *f,
- u32 crypto_op, u32 auth_op, u16 digest_len, u8 is_enc)
+ vec_reset_length (ptd->crypto_ops);
+ vec_reset_length (ptd->integ_ops);
+ vec_reset_length (ptd->chained_crypto_ops);
+ vec_reset_length (ptd->chained_integ_ops);
+ vec_reset_length (ptd->chunks);
+ fe = f->elts;
+ bi = f->buffer_indices;
+
+ while (n_elts--)
{
- vnet_crypto_async_frame_elt_t *fe;
- u32 *bi;
- u32 n_elts = f->n_elts;
- u8 state = VNET_CRYPTO_FRAME_STATE_SUCCESS;
-
- vec_reset_length (ptd->crypto_ops);
- vec_reset_length (ptd->integ_ops);
- vec_reset_length (ptd->chained_crypto_ops);
- vec_reset_length (ptd->chained_integ_ops);
- vec_reset_length (ptd->chunks);
- fe = f->elts;
- bi = f->buffer_indices;
-
- while (n_elts--)
- {
- if (n_elts > 1)
- clib_prefetch_load (fe + 1);
-
- crypto_sw_scheduler_convert_link_crypto (
- vm, ptd, cm->keys + fe->key_index, fe, fe - f->elts, bi[0],
- crypto_op, auth_op, digest_len, is_enc);
- bi++;
- fe++;
- }
+ if (n_elts > 1)
+ clib_prefetch_load (fe + 1);
- if (is_enc)
- {
- process_ops (vm, f, ptd->crypto_ops, &state);
- process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
- &state);
- process_ops (vm, f, ptd->integ_ops, &state);
- process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks,
- &state);
- }
- else
- {
- process_ops (vm, f, ptd->integ_ops, &state);
- process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks,
- &state);
- process_ops (vm, f, ptd->crypto_ops, &state);
- process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
- &state);
- }
+ crypto_sw_scheduler_convert_link_crypto (
+ vm, ptd, cm->keys + fe->key_index, fe, fe - f->elts, bi[0], crypto_op,
+ auth_op, digest_len, is_enc);
+ bi++;
+ fe++;
+ }
- f->state = state;
+ if (is_enc)
+ {
+ process_ops (vm, f, ptd->crypto_ops, &state);
+ process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
+ &state);
+ process_ops (vm, f, ptd->integ_ops, &state);
+ process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks, &state);
+ }
+ else
+ {
+ process_ops (vm, f, ptd->integ_ops, &state);
+ process_chained_ops (vm, f, ptd->chained_integ_ops, ptd->chunks, &state);
+ process_ops (vm, f, ptd->crypto_ops, &state);
+ process_chained_ops (vm, f, ptd->chained_crypto_ops, ptd->chunks,
+ &state);
}
- static_always_inline int
- convert_async_crypto_id (vnet_crypto_async_op_id_t async_op_id,
- u32 *crypto_op, u32 *auth_op_or_aad_len,
- u16 *digest_len, u8 *is_enc)
+ f->state = state;
+}
+
+static_always_inline int
+convert_async_crypto_id (vnet_crypto_async_op_id_t async_op_id, u32 *crypto_op,
+ u32 *auth_op_or_aad_len, u16 *digest_len, u8 *is_enc)
+{
+ switch (async_op_id)
{
- switch (async_op_id)
- {
#define _(n, s, k, t, a) \
case VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC: \
*crypto_op = VNET_CRYPTO_OP_##n##_ENC; \
*digest_len = t; \
*is_enc = 0; \
return 1;
- foreach_crypto_aead_async_alg
+ foreach_crypto_aead_async_alg
#undef _
#define _(c, h, s, k, d) \
*digest_len = d; \
*is_enc = 0; \
return 0;
- foreach_crypto_link_async_alg
+ foreach_crypto_link_async_alg
#undef _
- default : return -1;
- }
-
- return -1;
+ default : return -1;
}
- static_always_inline vnet_crypto_async_frame_t *
- crypto_sw_scheduler_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
- u32 *enqueue_thread_idx)
- {
- crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
- crypto_sw_scheduler_per_thread_data_t *ptd =
- cm->per_thread_data + vm->thread_index;
- vnet_crypto_async_frame_t *f = 0;
- crypto_sw_scheduler_queue_t *current_queue = 0;
- u32 tail, head;
- u8 found = 0;
-
- /* get a pending frame to process */
- if (ptd->self_crypto_enabled)
- {
- u32 i = ptd->last_serve_lcore_id + 1;
+ return -1;
+}
- while (1)
- {
- crypto_sw_scheduler_per_thread_data_t *st;
- u32 j;
+static_always_inline vnet_crypto_async_frame_t *
+crypto_sw_scheduler_dequeue (vlib_main_t *vm, u32 *nb_elts_processed,
+ u32 *enqueue_thread_idx)
+{
+ crypto_sw_scheduler_main_t *cm = &crypto_sw_scheduler_main;
+ crypto_sw_scheduler_per_thread_data_t *ptd =
+ cm->per_thread_data + vm->thread_index;
+ vnet_crypto_async_frame_t *f = 0;
+ crypto_sw_scheduler_queue_t *current_queue = 0;
+ u32 tail, head;
+ u8 found = 0;
+ u8 recheck_queues = 1;
+
+run_next_queues:
+ /* get a pending frame to process */
+ if (ptd->self_crypto_enabled)
+ {
+ u32 i = ptd->last_serve_lcore_id + 1;
- if (i >= vec_len (cm->per_thread_data))
- i = 0;
+ while (1)
+ {
+ crypto_sw_scheduler_per_thread_data_t *st;
+ u32 j;
- st = cm->per_thread_data + i;
+ if (i >= vec_len (cm->per_thread_data))
+ i = 0;
- if (ptd->last_serve_encrypt)
- current_queue = &st->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_DECRYPT];
- else
- current_queue = &st->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_ENCRYPT];
+ st = cm->per_thread_data + i;
- tail = current_queue->tail;
- head = current_queue->head;
+ if (ptd->last_serve_encrypt)
+ current_queue = &st->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_DECRYPT];
+ else
+ current_queue = &st->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_ENCRYPT];
- /* Skip this queue unless tail < head or head has overflowed
- * and tail has not. At the point where tail overflows (== 0),
- * the largest possible value of head is (queue size - 1).
- * Prior to that, the largest possible value of head is
- * (queue size - 2).
- */
- if ((tail > head) && (head >= CRYPTO_SW_SCHEDULER_QUEUE_MASK))
- goto skip_queue;
+ tail = current_queue->tail;
+ head = current_queue->head;
- for (j = tail; j != head; j++)
- {
+ /* Skip this queue unless tail < head or head has overflowed
+ * and tail has not. At the point where tail overflows (== 0),
+ * the largest possible value of head is (queue size - 1).
+ * Prior to that, the largest possible value of head is
+ * (queue size - 2).
+ */
+ if ((tail > head) && (head >= CRYPTO_SW_SCHEDULER_QUEUE_MASK))
+ goto skip_queue;
- f = current_queue->jobs[j & CRYPTO_SW_SCHEDULER_QUEUE_MASK];
+ for (j = tail; j != head; j++)
+ {
- if (!f)
- continue;
+ f = current_queue->jobs[j & CRYPTO_SW_SCHEDULER_QUEUE_MASK];
- if (clib_atomic_bool_cmp_and_swap (
- &f->state, VNET_CRYPTO_FRAME_STATE_PENDING,
- VNET_CRYPTO_FRAME_STATE_WORK_IN_PROGRESS))
- {
- found = 1;
- break;
- }
- }
+ if (!f)
+ continue;
- skip_queue:
- if (found || i == ptd->last_serve_lcore_id)
+ if (clib_atomic_bool_cmp_and_swap (
+ &f->state, VNET_CRYPTO_FRAME_STATE_PENDING,
+ VNET_CRYPTO_FRAME_STATE_WORK_IN_PROGRESS))
{
- CLIB_MEMORY_STORE_BARRIER ();
- ptd->last_serve_encrypt = !ptd->last_serve_encrypt;
+ found = 1;
break;
}
+ }
- i++;
+ skip_queue:
+ if (found || i == ptd->last_serve_lcore_id)
+ {
+ CLIB_MEMORY_STORE_BARRIER ();
+ ptd->last_serve_encrypt = !ptd->last_serve_encrypt;
+ break;
}
- ptd->last_serve_lcore_id = i;
+ i++;
}
- if (found)
- {
- u32 crypto_op, auth_op_or_aad_len;
- u16 digest_len;
- u8 is_enc;
- int ret;
-
- ret = convert_async_crypto_id (
- f->op, &crypto_op, &auth_op_or_aad_len, &digest_len, &is_enc);
-
- if (ret == 1)
- crypto_sw_scheduler_process_aead (vm, ptd, f, crypto_op,
- auth_op_or_aad_len, digest_len);
- else if (ret == 0)
- crypto_sw_scheduler_process_link (vm, cm, ptd, f, crypto_op,
- auth_op_or_aad_len, digest_len,
- is_enc);
-
- *enqueue_thread_idx = f->enqueue_thread_index;
- *nb_elts_processed = f->n_elts;
- }
+ ptd->last_serve_lcore_id = i;
+ }
- if (ptd->last_return_queue)
- {
- current_queue = &ptd->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_DECRYPT];
- ptd->last_return_queue = 0;
- }
- else
- {
- current_queue = &ptd->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_ENCRYPT];
- ptd->last_return_queue = 1;
- }
+ if (found)
+ {
+ u32 crypto_op, auth_op_or_aad_len;
+ u16 digest_len;
+ u8 is_enc;
+ int ret;
+
+ ret = convert_async_crypto_id (f->op, &crypto_op, &auth_op_or_aad_len,
+ &digest_len, &is_enc);
+
+ if (ret == 1)
+ crypto_sw_scheduler_process_aead (vm, ptd, f, crypto_op,
+ auth_op_or_aad_len, digest_len);
+ else if (ret == 0)
+ crypto_sw_scheduler_process_link (
+ vm, cm, ptd, f, crypto_op, auth_op_or_aad_len, digest_len, is_enc);
+
+ *enqueue_thread_idx = f->enqueue_thread_index;
+ *nb_elts_processed = f->n_elts;
+ }
+
+ if (ptd->last_return_queue)
+ {
+ current_queue = &ptd->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_DECRYPT];
+ ptd->last_return_queue = 0;
+ }
+ else
+ {
+ current_queue = &ptd->queue[CRYPTO_SW_SCHED_QUEUE_TYPE_ENCRYPT];
+ ptd->last_return_queue = 1;
+ }
- tail = current_queue->tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK;
+ tail = current_queue->tail & CRYPTO_SW_SCHEDULER_QUEUE_MASK;
- if (current_queue->jobs[tail] &&
- current_queue->jobs[tail]->state >= VNET_CRYPTO_FRAME_STATE_SUCCESS)
- {
+ if (current_queue->jobs[tail] &&
+ current_queue->jobs[tail]->state >= VNET_CRYPTO_FRAME_STATE_SUCCESS)
+ {
- CLIB_MEMORY_STORE_BARRIER ();
- current_queue->tail++;
- f = current_queue->jobs[tail];
- current_queue->jobs[tail] = 0;
+ CLIB_MEMORY_STORE_BARRIER ();
+ current_queue->tail++;
+ f = current_queue->jobs[tail];
+ current_queue->jobs[tail] = 0;
- return f;
- }
+ return f;
+ }
- return 0;
+ if (!found && recheck_queues)
+ {
+ recheck_queues = 0;
+ goto run_next_queues;
}
+ return 0;
+}
static clib_error_t *
sw_scheduler_set_worker_crypto (vlib_main_t * vm, unformat_input_t * input,
* @cliexstart{set sw_scheduler worker 0 crypto off}
* @cliexend
?*/
-/* *INDENT-OFF* */
VLIB_CLI_COMMAND (cmd_set_sw_scheduler_worker_crypto, static) = {
.path = "set sw_scheduler",
.short_help = "set sw_scheduler worker <idx> crypto <on|off>",
.function = sw_scheduler_set_worker_crypto,
.is_mp_safe = 1,
};
-/* *INDENT-ON* */
static clib_error_t *
sw_scheduler_show_workers (vlib_main_t * vm, unformat_input_t * input,
* @cliexstart{show sw_scheduler workers}
* @cliexend
?*/
-/* *INDENT-OFF* */
VLIB_CLI_COMMAND (cmd_show_sw_scheduler_workers, static) = {
.path = "show sw_scheduler workers",
.short_help = "show sw_scheduler workers",
.function = sw_scheduler_show_workers,
.is_mp_safe = 1,
};
-/* *INDENT-ON* */
clib_error_t *
sw_scheduler_cli_init (vlib_main_t * vm)
crypto_sw_scheduler_api_init (vm);
- /* *INDENT-OFF* */
#define _(n, s, k, t, a) \
vnet_crypto_register_enqueue_handler ( \
vm, cm->crypto_engine_index, VNET_CRYPTO_OP_##n##_TAG##t##_AAD##a##_ENC, \
crypto_sw_scheduler_frame_enqueue_decrypt);
foreach_crypto_link_async_alg
#undef _
- /* *INDENT-ON* */
vnet_crypto_register_dequeue_handler (vm, cm->crypto_engine_index,
crypto_sw_scheduler_dequeue);
return error;
}
-/* *INDENT-OFF* */
VLIB_INIT_FUNCTION (crypto_sw_scheduler_init) = {
.runs_after = VLIB_INITS ("vnet_crypto_init"),
};
.version = VPP_BUILD_VER,
.description = "SW Scheduler Crypto Async Engine plugin",
};
-/* *INDENT-ON* */
/*
* fd.io coding-style-patch-verification: ON
Code Review / vpp.git / blobdiff