2 * Copyright (c) 2018 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vnet/ipsec/ipsec.h>
19 #include <vlib/node_funcs.h>
20 #include <openssl/engine.h>
21 #include <tlsopenssl/tls_openssl.h>
23 #define MAX_SESSION 4096
24 #define MAX_VECTOR_ASYNC 256
26 #define SSL_ASYNC_INFLIGHT 1
27 #define SSL_ASYNC_PENDING 2
28 #define SSL_ASYNC_READY 3
30 #define EMPTY_STRUCT {0}
32 typedef struct openssl_tls_callback_arg_
36 } openssl_tls_callback_arg_t;
38 typedef struct openssl_event_
45 openssl_resume_handler *handler;
46 openssl_tls_callback_t engine_callback;
47 openssl_tls_callback_arg_t cb_args;
52 typedef struct openssl_async_status_
58 } openssl_async_status_t;
60 typedef struct openssl_async_
62 openssl_evt_t ***evt_pool;
63 openssl_async_status_t *status;
64 void (*polling) (void);
72 void dasync_polling ();
77 void (*polling) (void);
78 void (*pre_init) (void);
81 struct engine_polling engine_list[] = {
82 {"qat", qat_polling, qat_pre_init},
83 {"dasync", dasync_polling, NULL}
86 openssl_async_t openssl_async_main;
87 static vlib_node_registration_t tls_async_process_node;
89 /* to avoid build warning */
90 void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
94 evt_pool_init (vlib_main_t * vm)
96 vlib_thread_main_t *vtm = vlib_get_thread_main ();
97 openssl_async_t *om = &openssl_async_main;
100 num_threads = 1 /* main thread */ + vtm->n_threads;
102 TLS_DBG ("Totally there is %d thread\n", num_threads);
104 vec_validate (om->evt_pool, num_threads - 1);
105 vec_validate (om->status, num_threads - 1);
107 om->start_polling = 0;
110 for (i = 0; i < num_threads; i++)
112 om->status[i].evt_run_head = -1;
113 om->status[i].evt_run_tail = -1;
114 om->status[i].evt_pending_head = -1;
118 TLS_DBG ("Node disabled\n");
119 openssl_async_node_enable_disable (0);
125 openssl_engine_register (char *engine_name, char *algorithm)
127 int i, registered = -1;
128 openssl_async_t *om = &openssl_async_main;
132 for (i = 0; i < ARRAY_LEN (engine_list); i++)
134 if (!strcmp (engine_list[i].engine, engine_name))
136 om->polling = engine_list[i].polling;
146 ENGINE_load_builtin_engines ();
147 ENGINE_load_dynamic ();
148 engine = ENGINE_by_id (engine_name);
157 p = engine_list[registered].pre_init;
163 if (!ENGINE_set_default_string (engine, algorithm))
165 clib_warning ("Failed to set engine %s algorithm %s\n",
166 engine_name, algorithm);
172 if (!ENGINE_set_default (engine, ENGINE_METHOD_ALL))
174 clib_warning ("Failed to set engine %s to all algorithm",
180 om->start_polling = 1;
186 static openssl_evt_t *
187 openssl_evt_get (u32 evt_index)
191 pool_elt_at_index (openssl_async_main.evt_pool[vlib_get_thread_index ()],
196 static openssl_evt_t *
197 openssl_evt_get_w_thread (int evt_index, u8 thread_index)
202 pool_elt_at_index (openssl_async_main.evt_pool[thread_index], evt_index);
207 openssl_evt_free (int event_idx, u8 thread_index)
210 openssl_async_t *om = &openssl_async_main;
211 int *evt_run_tail = &om->status[thread_index].evt_run_tail;
216 evt = openssl_evt_get_w_thread (event_idx, thread_index);
221 pool_put_index (om->evt_pool[thread_index], event_idx);
223 if (*evt_run_tail == event_idx)
230 openssl_evt_alloc (void)
232 u8 thread_index = vlib_get_thread_index ();
233 openssl_async_t *tm = &openssl_async_main;
236 pool_get (tm->evt_pool[thread_index], evt);
238 *evt = clib_mem_alloc (sizeof (openssl_evt_t));
240 memset (*evt, 0, sizeof (openssl_evt_t));
241 (*evt)->event_index = evt - tm->evt_pool[thread_index];
242 return ((*evt)->event_index);
246 openssl_async_run (void *evt)
248 openssl_evt_t *event, *event_tail;
249 openssl_async_t *om = &openssl_async_main;
250 openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) evt;
251 int thread_index = args->thread_index;
252 int event_index = args->event_index;
253 int *evt_run_tail = &om->status[thread_index].evt_run_tail;
254 int *evt_run_head = &om->status[thread_index].evt_run_head;
256 TLS_DBG ("Set event %d to run\n", event_index);
258 event = openssl_evt_get_w_thread (event_index, thread_index);
260 if (event->status == SSL_ASYNC_READY)
263 event->status = SSL_ASYNC_READY;
267 if (*evt_run_tail >= 0)
269 event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index);
270 event_tail->next = event_index;
272 *evt_run_tail = event_index;
273 if (*evt_run_head < 0)
274 *evt_run_head = event_index;
279 openssl_tls_callback_t *
280 vpp_add_async_pending_event (tls_ctx_t * ctx,
281 openssl_resume_handler * handler)
284 openssl_evt_t *event;
285 openssl_async_t *om = &openssl_async_main;
286 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
287 int *evt_pending_head;
288 u32 thread_id = ctx->c_thread_index;
290 eidx = openssl_evt_alloc ();
291 event = openssl_evt_get (eidx);
293 event->ctx_index = oc->openssl_ctx_index;
294 event->status = SSL_ASYNC_PENDING;
295 event->handler = handler;
296 event->cb_args.event_index = eidx;
297 event->cb_args.thread_index = thread_id;
298 event->engine_callback.callback = openssl_async_run;
299 event->engine_callback.arg = &event->cb_args;
301 /* add to pending list */
302 evt_pending_head = &om->status[thread_id].evt_pending_head;
303 event->next = *evt_pending_head;
304 *evt_pending_head = eidx;
307 return &event->engine_callback;
311 event_handler (void *tls_async)
314 openssl_resume_handler *handler;
315 openssl_evt_t *callback;
316 stream_session_t *tls_session;
320 callback = (openssl_evt_t *) tls_async;
321 thread_index = callback->cb_args.thread_index;
322 ctx = openssl_ctx_get_w_thread (callback->ctx_index, thread_index);
323 handler = callback->handler;
324 tls_session = session_get_from_handle (ctx->tls_session_handle);
328 TLS_DBG ("relaunch...\n");
329 (*handler) (ctx, tls_session);
332 /* Need to free the event */
333 openssl_evt_free (callback->cb_args.event_index, thread_index);
338 /* engine specific code to polling the response ring */
342 openssl_async_t *om = &openssl_async_main;
343 openssl_evt_t *event;
345 openssl_tls_callback_t *engine_cb;
346 u8 thread_index = vlib_get_thread_index ();
348 /* POC code here to simulate the engine to call callback */
349 evt_pending = &om->status[thread_index].evt_pending_head;
350 while (*evt_pending >= 0)
352 TLS_DBG ("polling... current head = %d\n", *evt_pending);
353 event = openssl_evt_get_w_thread (*evt_pending, thread_index);
354 *evt_pending = event->next;
355 if (event->status == SSL_ASYNC_PENDING)
357 engine_cb = &event->engine_callback;
358 (*engine_cb->callback) (engine_cb->arg);
367 openssl_async_t *om = &openssl_async_main;
369 ENGINE_ctrl_cmd (om->engine, "ENABLE_EXTERNAL_POLLING", 0, NULL, NULL, 0);
372 /* Below code is spefic to QAT engine, and other vendors can refer to this code to enable a new engine */
374 qat_polling_config ()
376 openssl_async_t *om = &openssl_async_main;
377 u8 thread_index = vlib_get_thread_index ();
380 config = &om->status[thread_index].poll_config;
384 ENGINE_ctrl_cmd (om->engine, "SET_INSTANCE_FOR_THREAD", thread_index,
388 TLS_DBG ("set thread %d and instance %d mapping\n", thread_index,
396 openssl_async_t *om = &openssl_async_main;
399 if (om->start_polling)
401 qat_polling_config ();
402 #define QAT_CMD_POLL (ENGINE_CMD_BASE + 1)
403 ENGINE_ctrl (om->engine, QAT_CMD_POLL, 0, &ret, NULL);
409 openssl_async_polling ()
411 openssl_async_t *om = &openssl_async_main;
419 openssl_async_node_enable_disable (u8 is_en)
421 u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
423 foreach_vlib_main (({
424 vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
431 tls_async_do_job (int eidx, u32 thread_index)
434 openssl_evt_t *event;
436 /* do the real job */
437 event = openssl_evt_get_w_thread (eidx, thread_index);
438 ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index);
443 session_send_rpc_evt_to_thread (thread_index, event_handler, event);
449 tls_resume_from_crypto (int thread_index)
453 openssl_async_t *om = &openssl_async_main;
454 openssl_evt_t *event;
455 int *evt_run_head = &om->status[thread_index].evt_run_head;
457 if (*evt_run_head < 0)
460 for (i = 0; i < MAX_VECTOR_ASYNC; i++)
462 if (*evt_run_head >= 0)
464 event = openssl_evt_get_w_thread (*evt_run_head, thread_index);
465 TLS_DBG ("event run = %d\n", *evt_run_head);
466 tls_async_do_job (*evt_run_head, thread_index);
468 *evt_run_head = event->next;
481 static clib_error_t *
482 tls_async_init (vlib_main_t * vm)
485 TLS_DBG ("Start to call tls_async_init\n");
492 tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
497 thread_index = vlib_get_thread_index ();
498 openssl_async_polling ();
500 tls_resume_from_crypto (thread_index);
505 VLIB_INIT_FUNCTION (tls_async_init);
508 VLIB_REGISTER_NODE (tls_async_process_node,static) = {
509 .function = tls_async_process,
510 .type = VLIB_NODE_TYPE_INPUT,
511 .name = "tls-async-process",
518 * fd.io coding-style-patch-verification: ON
521 * eval: (c-set-style "gnu")