odp/thread.c \
odp/ipsec/ipsec.c \
odp/ipsec/esp_encrypt.c \
- odp/ipsec/esp_decrypt.c
+ odp/ipsec/esp_decrypt.c \
+ odp/ipsec/crypto_input.c
noinst_HEADERS += odp/odp_packet.h \
odp/ipsec/ipsec.h \
--- /dev/null
+#include <vlib/vlib.h>
+#include <vnet/ip/ip.h>
+
+#include <odp/ipsec/ipsec.h>
+#include <odp/ipsec/esp.h>
+#include <odp/odp_packet.h>
+
+#include <assert.h>
+
+#define foreach_odp_crypto_input_next \
+ _(DROP, "error-drop") \
+ _(ENCRYPT_POST, "odp-crypto-esp-encrypt-post") \
+ _(DECRYPT_POST, "odp-crypto-esp-decrypt-post")
+
+typedef enum
+{
+#define _(f, s) ODP_CRYPTO_INPUT_NEXT_##f,
+ foreach_odp_crypto_input_next
+#undef _
+ODP_CRYPTO_INPUT_N_NEXT,
+} odp_crypto_input_next_t;
+
+#define foreach_crypto_input_error \
+_(DEQUE_COP, "Dequed crypto operations")
+
+typedef enum
+{
+#define _(sym,str) CRYPTO_INPUT_ERROR_##sym,
+ foreach_crypto_input_error
+#undef _
+} crypto_input_error_t;
+
+static char *crypto_input_error_strings[] = {
+#define _(sym,string) string,
+ foreach_crypto_input_error
+#undef _
+};
+
+typedef struct
+{
+ u32 next_index;
+ odp_packet_t pkt;
+} odp_packet_crypto_trace_t;
+
+static u8 *
+format_odp_crypto_input_trace (u8 * s, va_list * args)
+{
+ CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+ CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+
+ s = format (s, " odp-crypto-input ");
+
+ return s;
+}
+
+static uword
+odp_dequeue_cops (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * frame, odp_queue_t queue, u32 next_node_index)
+{
+ u32 next_index = next_node_index, n_deq, n_cops, *to_next = 0;
+ const int MAX_EVENTS = (1<<8);
+ odp_event_t events[MAX_EVENTS];
+
+ n_deq = odp_queue_deq_multi(queue, events, MAX_EVENTS);
+
+ n_cops = n_deq;
+
+ int index = 0;
+ while(n_cops > 0) {
+ u32 n_left_to_next;
+
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+ while(n_cops > 0 && n_left_to_next > 0) {
+ odp_event_t event = events[index++];
+
+ ASSERT(ODP_EVENT_CRYPTO_COMPL == odp_event_type(event));
+
+ odp_crypto_compl_t compl;
+ odp_crypto_op_result_t result;
+ odp_packet_t pkt;
+ vlib_buffer_t *b0;
+ u32 bi0;
+
+ compl = odp_crypto_compl_from_event(event);
+ odp_crypto_compl_result(compl, &result);
+ pkt = result.pkt;
+
+ b0 = vlib_buffer_from_odp_packet(pkt);
+ bi0 = vlib_get_buffer_index (vm, b0);
+
+ to_next[0] = bi0;
+ to_next += 1;
+
+ n_cops -= 1;
+ n_left_to_next -= 1;
+
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ {
+ odp_packet_crypto_trace_t *tr;
+ tr = vlib_add_trace (vm, node, b0, sizeof (*tR));
+ tr->next_index = next_index;
+ }
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+ to_next, n_left_to_next, bi0, next_node_index);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+ vlib_node_increment_counter (vm, odp_crypto_input_node.index,
+ CRYPTO_INPUT_ERROR_DEQUE_COP,
+ n_deq);
+
+ return n_deq;
+}
+
+static uword
+odp_crypto_input_node_fn (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * frame)
+{
+ odp_crypto_main_t *ocm = &odp_crypto_main;
+ u32 thread_index = vlib_get_thread_index ();
+ odp_crypto_worker_main_t *cwm =
+ vec_elt_at_index (ocm->workers, thread_index);
+ u32 n_cops_dequeued = 0;
+ n_cops_dequeued += odp_dequeue_cops(vm, node, frame, cwm->post_encrypt, ODP_CRYPTO_INPUT_NEXT_ENCRYPT_POST);
+ n_cops_dequeued += odp_dequeue_cops(vm, node, frame, cwm->post_decrypt, ODP_CRYPTO_INPUT_NEXT_DECRYPT_POST);
+ return n_cops_dequeued;
+}
+
+
+VLIB_REGISTER_NODE (odp_crypto_input_node) =
+{
+ .function = odp_crypto_input_node_fn,
+ .name = "odp-crypto-input",
+ .format_trace = format_odp_crypto_input_trace,
+ .type = VLIB_NODE_TYPE_INPUT,
+ .state = VLIB_NODE_STATE_DISABLED,
+
+ .n_errors = ARRAY_LEN(crypto_input_error_strings),
+ .error_strings = crypto_input_error_strings,
+
+ .n_next_nodes = ODP_CRYPTO_INPUT_N_NEXT,
+ .next_nodes =
+ {
+#define _(s, n) [ODP_CRYPTO_INPUT_NEXT_##s] = n,
+ foreach_odp_crypto_input_next
+#undef _
+ },
+};
+
+VLIB_NODE_FUNCTION_MULTIARCH (odp_crypto_input_node, odp_crypto_input_node_fn);
+
+
+
} esp_decrypt_trace_t;
vlib_node_registration_t odp_crypto_esp_decrypt_node;
+vlib_node_registration_t odp_crypto_esp_decrypt_post_node;
/* packet trace format function */
static u8 *
return s;
}
+/* packet trace format function */
+static u8 *
+format_esp_decrypt_post_trace (u8 * s, va_list * args)
+{
+ CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+ CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+
+ s = format (s, "POST DECRYPT CRYPTO (ODP)");
+ return s;
+}
+
static uword
esp_decrypt_node_fn (vlib_main_t * vm,
vlib_node_runtime_t * node, vlib_frame_t * from_frame)
while (n_left_from > 0)
{
u32 n_left_to_next;
+ u32 buffers_passed = 0;
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
u8 tunnel_mode = 1;
u8 transport_ip6 = 0;
sa_data_t *sa_sess_data;
-
+ odp_crypto_op_param_t crypto_op_params;
+ odp_crypto_op_result_t result;
+ odp_bool_t posted = 0;
bi0 = from[0];
from += 1;
}
}
- odp_crypto_op_param_t crypto_op_params;
- odp_bool_t posted = 0;
- odp_crypto_op_result_t result;
-
crypto_op_params.session = sa_sess_data->sess;
crypto_op_params.ctx = NULL;
crypto_op_params.aad.ptr = NULL;
next0 = ESP_DECRYPT_NEXT_IPSEC_GRE_INPUT;
vnet_buffer (b0)->sw_if_index[VLIB_TX] = (u32) ~ 0;
+
+ vnet_buffer (b0)->post_crypto.next_index = next0;
}
trace:
if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
{
- b0->flags |= VLIB_BUFFER_IS_TRACED;
- b0->trace_index = b0->trace_index;
esp_decrypt_trace_t *tr =
vlib_add_trace (vm, node, b0, sizeof (*tr));
tr->crypto_alg = sa0->crypto_alg;
tr->integ_alg = sa0->integ_alg;
}
- vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
- n_left_to_next, bi0, next0);
+ if (!posted)
+ {
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
+ n_left_to_next, bi0, next0);
+ buffers_passed += 1;
+ }
}
- vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ if (buffers_passed > 0)
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
}
vlib_node_increment_counter (vm, odp_crypto_esp_decrypt_node.index,
ESP_DECRYPT_ERROR_RX_PKTS,
/* *INDENT-ON* */
VLIB_NODE_FUNCTION_MULTIARCH (odp_crypto_esp_decrypt_node, esp_decrypt_node_fn)
+ static uword
+ esp_decrypt_post_node_fn (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * from_frame)
+{
+ u32 n_left_from, *from, *to_next = 0, next_index;
+ from = vlib_frame_vector_args (from_frame);
+ n_left_from = from_frame->n_vectors;
+
+ next_index = node->cached_next_index;
+
+ while (n_left_from > 0)
+ {
+ u32 n_left_to_next;
+
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ u32 bi0, next0;
+ vlib_buffer_t *b0 = 0;
+
+ bi0 = from[0];
+ from += 1;
+ n_left_from -= 1;
+ n_left_to_next -= 1;
+
+ b0 = vlib_get_buffer (vm, bi0);
+
+ to_next[0] = bi0;
+ to_next += 1;
+
+ next0 = vnet_buffer (b0)->post_crypto.next_index;
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+ to_next, n_left_to_next, bi0,
+ next0);
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ vlib_add_trace (vm, node, b0, 0);
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+
+ }
+ vlib_node_increment_counter (vm, odp_crypto_esp_decrypt_post_node.index,
+ ESP_DECRYPT_ERROR_RX_PKTS,
+ from_frame->n_vectors);
+
+ return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (odp_crypto_esp_decrypt_post_node) = {
+ .function = esp_decrypt_post_node_fn,
+ .name = "odp-crypto-esp-decrypt-post",
+ .vector_size = sizeof (u32),
+ .format_trace = format_esp_decrypt_post_trace,
+ .type = VLIB_NODE_TYPE_INTERNAL,
+
+ .n_errors = ARRAY_LEN(esp_decrypt_error_strings),
+ .error_strings = esp_decrypt_error_strings,
+
+ .n_next_nodes = ESP_DECRYPT_N_NEXT,
+ .next_nodes = {
+#define _(s,n) [ESP_DECRYPT_NEXT_##s] = n,
+ foreach_esp_decrypt_next
+#undef _
+ },
+};
+/* *INDENT-ON* */
+
/*
* fd.io coding-style-patch-verification: ON
*
} esp_encrypt_trace_t;
vlib_node_registration_t odp_crypto_esp_encrypt_node;
+vlib_node_registration_t odp_crypto_esp_encrypt_post_node;
/* packet trace format function */
static u8 *
return s;
}
+/* packet trace format function */
+static u8 *
+format_esp_encrypt_post_trace (u8 * s, va_list * args)
+{
+ CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
+ CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
+
+ s = format (s, "POST ENCRYPT CRYPTO (ODP) esp");
+ return s;
+}
+
+
static uword
esp_encrypt_node_fn (vlib_main_t * vm,
vlib_node_runtime_t * node, vlib_frame_t * from_frame)
while (n_left_from > 0)
{
u32 n_left_to_next;
+ u32 buffers_passed = 0;
vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
u32 ip_proto = 0;
u8 transport_mode = 0;
sa_data_t *sa_sess_data;
+ odp_bool_t posted = 0;
bi0 = from[0];
from += 1;
BLOCK_SIZE * blocks + sizeof (esp_header_t) + IV_SIZE;
odp_crypto_op_param_t crypto_op_params;
- odp_bool_t posted = 0;
odp_crypto_op_result_t result;
crypto_op_params.session = sa_sess_data->sess;
b0->current_length +=
em->esp_integ_algs[sa0->integ_alg].trunc_size;
+ vnet_buffer (b0)->post_crypto.next_index = next0;
+
int ret =
odp_crypto_operation (&crypto_op_params, &posted, &result);
if (ret != 0)
trace:
if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
{
- b0->flags |= VLIB_BUFFER_IS_TRACED;
- b0->trace_index = b0->trace_index;
esp_encrypt_trace_t *tr =
vlib_add_trace (vm, node, b0, sizeof (*tr));
tr->spi = sa0->spi;
tr->integ_alg = sa0->integ_alg;
}
- vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
- to_next, n_left_to_next, bi0,
- next0);
+ if (!posted)
+ {
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+ to_next, n_left_to_next, bi0,
+ next0);
+ buffers_passed += 1;
+ }
+
}
- vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ if (buffers_passed > 0)
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
}
vlib_node_increment_counter (vm, odp_crypto_esp_encrypt_node.index,
ESP_ENCRYPT_ERROR_RX_PKTS,
/* *INDENT-ON* */
VLIB_NODE_FUNCTION_MULTIARCH (odp_crypto_esp_encrypt_node, esp_encrypt_node_fn)
+ static uword
+ esp_encrypt_post_node_fn (vlib_main_t * vm,
+ vlib_node_runtime_t * node,
+ vlib_frame_t * from_frame)
+{
+ u32 n_left_from, *from, *to_next = 0, next_index;
+ from = vlib_frame_vector_args (from_frame);
+ n_left_from = from_frame->n_vectors;
+
+ next_index = node->cached_next_index;
+
+ while (n_left_from > 0)
+ {
+ u32 n_left_to_next;
+
+ vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+
+ while (n_left_from > 0 && n_left_to_next > 0)
+ {
+ u32 bi0, next0;
+ vlib_buffer_t *b0 = 0;
+
+ bi0 = from[0];
+ from += 1;
+ n_left_from -= 1;
+ n_left_to_next -= 1;
+
+ b0 = vlib_get_buffer (vm, bi0);
+
+ to_next[0] = bi0;
+ to_next += 1;
+
+ next0 = vnet_buffer (b0)->post_crypto.next_index;
+
+ vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
+ to_next, n_left_to_next, bi0,
+ next0);
+ if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
+ vlib_add_trace (vm, node, b0, 0);
+
+ }
+ vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ }
+ vlib_node_increment_counter (vm, odp_crypto_esp_encrypt_post_node.index,
+ ESP_ENCRYPT_ERROR_RX_PKTS,
+ from_frame->n_vectors);
+
+ return from_frame->n_vectors;
+}
+
+/* *INDENT-OFF* */
+VLIB_REGISTER_NODE (odp_crypto_esp_encrypt_post_node) = {
+ .function = esp_encrypt_post_node_fn,
+ .name = "odp-crypto-esp-encrypt-post",
+ .vector_size = sizeof (u32),
+ .format_trace = format_esp_encrypt_post_trace,
+ .type = VLIB_NODE_TYPE_INTERNAL,
+
+ .n_errors = ARRAY_LEN(esp_encrypt_error_strings),
+ .error_strings = esp_encrypt_error_strings,
+
+ .n_next_nodes = ESP_ENCRYPT_N_NEXT,
+ .next_nodes = {
+#define _(s,n) [ESP_ENCRYPT_NEXT_##s] = n,
+ foreach_esp_encrypt_next
+#undef _
+ },
+};
+/* *INDENT-ON* */
+
/*
* fd.io coding-style-patch-verification: ON
*
odp_crypto_session_param_t crypto_params;
odp_crypto_session_param_init (&crypto_params);
+ odp_crypto_main_t *ocm = &odp_crypto_main;
+ u32 thread_index = vlib_get_thread_index ();
+ odp_crypto_worker_main_t *cwm =
+ vec_elt_at_index (ocm->workers, thread_index);
+
esp_main_t *em = &odp_esp_main;
int trunc_size = em->esp_integ_algs[sa->integ_alg].trunc_size;
crypto_params.auth_cipher_text = 1;
- /* Synchronous mode */
- crypto_params.pref_mode = ODP_CRYPTO_SYNC;
- crypto_params.compl_queue = ODP_QUEUE_INVALID;
+ crypto_params.pref_mode = ODP_CRYPTO_ASYNC;
+ crypto_params.compl_queue =
+ (is_outbound ? cwm->post_encrypt : cwm->post_decrypt);
crypto_params.output_pool = ODP_POOL_INVALID;
if (sa->crypto_alg == IPSEC_CRYPTO_ALG_AES_CBC_128)
odp_crypto_main_t *ocm = &odp_crypto_main;
vlib_thread_main_t *tm = vlib_get_thread_main ();
vlib_node_t *ipsec_node, *crypto_node, *error_node;
+ odp_crypto_worker_main_t *cwm;
memset (im, 0, sizeof (im[0]));
vec_alloc (ocm->workers, tm->n_vlib_mains);
_vec_len (ocm->workers) = tm->n_vlib_mains;
+ for (cwm = ocm->workers + 1; cwm < vec_end (ocm->workers); cwm++)
+ {
+ cwm->post_encrypt = odp_queue_create (NULL, NULL);
+ cwm->post_decrypt = odp_queue_create (NULL, NULL);
+ }
+
esp_init ();
+ int i;
+ for (i = 1; i < tm->n_vlib_mains; i++)
+ vlib_node_set_state (vlib_mains[i], odp_crypto_input_node.index,
+ VLIB_NODE_STATE_POLLING);
+
+ /* If there are no worker threads, enable polling
+ crypto devices on the main thread, else
+ assign the post crypt queues of the second
+ thread to the main thread crypto sessions */
+ if (tm->n_vlib_mains == 1)
+ {
+ ocm->workers[0].post_encrypt = odp_queue_create (NULL, NULL);
+ ocm->workers[0].post_decrypt = odp_queue_create (NULL, NULL);
+ vlib_node_set_state (vlib_mains[0], odp_crypto_input_node.index,
+ VLIB_NODE_STATE_POLLING);
+ }
+ else
+ {
+ ocm->workers[0].post_encrypt = ocm->workers[1].post_encrypt;
+ ocm->workers[0].post_decrypt = ocm->workers[1].post_decrypt;
+ }
+
return 0;
}
#include <vlib/unix/unix.h>
#include <vnet/ip/ip.h>
#include <vnet/ethernet/ethernet.h>
+#include <odp/ipsec/ipsec.h>
#include <vnet/plugin/plugin.h>
#include <vpp/app/version.h>
#include <odp/odp_packet.h>
## Make use of ODP crypto API to accelerate IPsec
# enable-odp-crypto
+
+ ## To use asynchronous mode of the crypto operations
+ # async
# }
# Adjusting the plugin path depending on where the VPP plugins are:
Code Review / odp4vpp.git / commitdiff