2 * Copyright (c) 2018 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/vnet.h>
16 #include <vnet/ip/ip.h>
17 #include <vnet/api_errno.h>
18 #include <vlib/node_funcs.h>
19 #include <openssl/engine.h>
20 #include <tlsopenssl/tls_openssl.h>
22 #define MAX_SESSION 4096
23 #define MAX_VECTOR_ASYNC 256
25 #define SSL_ASYNC_INFLIGHT 1
26 #define SSL_ASYNC_PENDING 2
27 #define SSL_ASYNC_READY 3
29 #define EMPTY_STRUCT {0}
31 typedef struct openssl_tls_callback_arg_
35 } openssl_tls_callback_arg_t;
37 typedef struct openssl_event_
44 openssl_resume_handler *handler;
45 openssl_tls_callback_t engine_callback;
46 openssl_tls_callback_arg_t cb_args;
51 typedef struct openssl_async_status_
57 } openssl_async_status_t;
59 typedef struct openssl_async_
61 openssl_evt_t ***evt_pool;
62 openssl_async_status_t *status;
63 void (*polling) (void);
64 void (*polling_conf) (void);
72 void qat_polling_config ();
73 void dasync_polling ();
78 void (*polling) (void);
79 void (*pre_init) (void);
80 void (*polling_conf) (void);
83 struct engine_polling engine_list[] = {
84 {"qat", qat_polling, qat_pre_init, qat_polling_config},
85 {"dasync", dasync_polling, NULL, NULL}
88 openssl_async_t openssl_async_main;
89 static vlib_node_registration_t tls_async_process_node;
91 /* to avoid build warning */
92 void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
96 evt_pool_init (vlib_main_t * vm)
98 vlib_thread_main_t *vtm = vlib_get_thread_main ();
99 openssl_async_t *om = &openssl_async_main;
102 num_threads = 1 /* main thread */ + vtm->n_threads;
104 TLS_DBG (2, "Totally there is %d thread\n", num_threads);
106 vec_validate (om->evt_pool, num_threads - 1);
107 vec_validate (om->status, num_threads - 1);
109 om->start_polling = 0;
112 for (i = 0; i < num_threads; i++)
114 om->status[i].evt_run_head = -1;
115 om->status[i].evt_run_tail = -1;
116 om->status[i].evt_pending_head = -1;
120 openssl_async_node_enable_disable (0);
126 openssl_engine_register (char *engine_name, char *algorithm)
128 int i, registered = -1;
129 openssl_async_t *om = &openssl_async_main;
133 for (i = 0; i < ARRAY_LEN (engine_list); i++)
135 if (!strcmp (engine_list[i].engine, engine_name))
137 om->polling = engine_list[i].polling;
138 om->polling_conf = engine_list[i].polling_conf;
145 clib_error ("engine %s is not regisered in VPP", engine_name);
149 ENGINE_load_builtin_engines ();
150 ENGINE_load_dynamic ();
151 engine = ENGINE_by_id (engine_name);
155 clib_warning ("Failed to find engine ENGINE_by_id %s", engine_name);
161 p = engine_list[registered].pre_init;
167 if (!ENGINE_set_default_string (engine, algorithm))
169 clib_warning ("Failed to set engine %s algorithm %s\n",
170 engine_name, algorithm);
176 if (!ENGINE_set_default (engine, ENGINE_METHOD_ALL))
178 clib_warning ("Failed to set engine %s to all algorithm",
184 om->start_polling = 1;
190 static openssl_evt_t *
191 openssl_evt_get (u32 evt_index)
195 pool_elt_at_index (openssl_async_main.evt_pool[vlib_get_thread_index ()],
200 static openssl_evt_t *
201 openssl_evt_get_w_thread (int evt_index, u8 thread_index)
206 pool_elt_at_index (openssl_async_main.evt_pool[thread_index], evt_index);
211 openssl_evt_free (int event_idx, u8 thread_index)
214 openssl_async_t *om = &openssl_async_main;
215 int *evt_run_tail = &om->status[thread_index].evt_run_tail;
220 evt = openssl_evt_get_w_thread (event_idx, thread_index);
225 pool_put_index (om->evt_pool[thread_index], event_idx);
227 if (*evt_run_tail == event_idx)
234 openssl_evt_alloc (void)
236 u8 thread_index = vlib_get_thread_index ();
237 openssl_async_t *tm = &openssl_async_main;
240 pool_get (tm->evt_pool[thread_index], evt);
242 *evt = clib_mem_alloc (sizeof (openssl_evt_t));
244 clib_memset (*evt, 0, sizeof (openssl_evt_t));
245 (*evt)->event_index = evt - tm->evt_pool[thread_index];
246 return ((*evt)->event_index);
250 tls_async_openssl_callback (SSL * s, void *evt)
252 openssl_evt_t *event, *event_tail;
253 openssl_async_t *om = &openssl_async_main;
254 openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) evt;
255 int thread_index = args->thread_index;
256 int event_index = args->event_index;
257 int *evt_run_tail = &om->status[thread_index].evt_run_tail;
258 int *evt_run_head = &om->status[thread_index].evt_run_head;
260 TLS_DBG (2, "Set event %d to run\n", event_index);
262 event = openssl_evt_get_w_thread (event_index, thread_index);
264 if (event->status == SSL_ASYNC_READY)
267 event->status = SSL_ASYNC_READY;
271 if (*evt_run_tail >= 0)
273 event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index);
274 event_tail->next = event_index;
276 *evt_run_tail = event_index;
277 if (*evt_run_head < 0)
279 *evt_run_head = event_index;
285 openssl_tls_callback_t *
286 vpp_add_async_pending_event (tls_ctx_t * ctx,
287 openssl_resume_handler * handler)
290 openssl_evt_t *event;
291 openssl_async_t *om = &openssl_async_main;
292 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
293 int *evt_pending_head;
294 u32 thread_id = ctx->c_thread_index;
296 eidx = openssl_evt_alloc ();
297 event = openssl_evt_get (eidx);
299 event->ctx_index = oc->openssl_ctx_index;
300 event->status = SSL_ASYNC_PENDING;
301 event->handler = handler;
302 event->cb_args.event_index = eidx;
303 event->cb_args.thread_index = thread_id;
304 event->engine_callback.callback = tls_async_openssl_callback;
305 event->engine_callback.arg = &event->cb_args;
307 /* add to pending list */
308 evt_pending_head = &om->status[thread_id].evt_pending_head;
309 event->next = *evt_pending_head;
310 *evt_pending_head = eidx;
312 return &event->engine_callback;
316 vpp_add_async_run_event (tls_ctx_t * ctx, openssl_resume_handler * handler)
319 openssl_evt_t *event;
320 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
321 u32 thread_id = ctx->c_thread_index;
323 eidx = openssl_evt_alloc ();
324 event = openssl_evt_get (eidx);
326 event->ctx_index = oc->openssl_ctx_index;
327 event->status = SSL_ASYNC_PENDING;
328 event->handler = handler;
329 event->cb_args.event_index = eidx;
330 event->cb_args.thread_index = thread_id;
331 event->engine_callback.callback = tls_async_openssl_callback;
332 event->engine_callback.arg = &event->cb_args;
334 /* This is a retry event, and need to put to ring to make it run again */
335 return tls_async_openssl_callback (NULL, &event->cb_args);
340 event_handler (void *tls_async)
343 openssl_resume_handler *handler;
344 openssl_evt_t *callback;
345 session_t *tls_session;
349 callback = (openssl_evt_t *) tls_async;
350 thread_index = callback->cb_args.thread_index;
351 ctx = openssl_ctx_get_w_thread (callback->ctx_index, thread_index);
352 handler = callback->handler;
353 tls_session = session_get_from_handle (ctx->tls_session_handle);
357 (*handler) (ctx, tls_session);
360 /* Need to free the event */
361 openssl_evt_free (callback->cb_args.event_index, thread_index);
366 /* engine specific code to polling the response ring */
370 /* dasync is a fake async device, and could not be polled.
371 * We have added code in the dasync engine to triggered the callback already,
372 * so nothing can be done here
379 openssl_async_t *om = &openssl_async_main;
381 ENGINE_ctrl_cmd (om->engine, "ENABLE_EXTERNAL_POLLING", 0, NULL, NULL, 0);
384 /* Below code is spefic to QAT engine, and other vendors can refer to this code to enable a new engine */
386 qat_polling_config ()
388 openssl_async_t *om = &openssl_async_main;
389 u8 thread_index = vlib_get_thread_index ();
392 config = &om->status[thread_index].poll_config;
393 if (PREDICT_TRUE (*config))
396 ENGINE_ctrl_cmd (om->engine, "SET_INSTANCE_FOR_THREAD", thread_index,
400 TLS_DBG (2, "set thread %d and instance %d mapping\n", thread_index,
408 openssl_async_t *om = &openssl_async_main;
411 if (om->start_polling)
413 ENGINE_ctrl_cmd (om->engine, "POLL", 0, &poll_status, NULL, 0);
418 openssl_async_polling ()
420 openssl_async_t *om = &openssl_async_main;
428 openssl_async_node_enable_disable (u8 is_en)
430 u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
432 foreach_vlib_main (({
433 vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
440 tls_async_do_job (int eidx, u32 thread_index)
443 openssl_evt_t *event;
445 /* do the real job */
446 event = openssl_evt_get_w_thread (eidx, thread_index);
447 ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index);
452 session_send_rpc_evt_to_thread (thread_index, event_handler, event);
458 tls_resume_from_crypto (int thread_index)
462 openssl_async_t *om = &openssl_async_main;
463 openssl_evt_t *event;
464 int *evt_run_head = &om->status[thread_index].evt_run_head;
466 if (*evt_run_head < 0)
469 for (i = 0; i < MAX_VECTOR_ASYNC; i++)
471 if (*evt_run_head >= 0)
473 event = openssl_evt_get_w_thread (*evt_run_head, thread_index);
474 TLS_DBG (2, "event run = %d\n", *evt_run_head);
475 tls_async_do_job (*evt_run_head, thread_index);
477 *evt_run_head = event->next;
490 static clib_error_t *
491 tls_async_init (vlib_main_t * vm)
499 tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
503 openssl_async_t *om = &openssl_async_main;
505 if (om->polling_conf)
506 (*om->polling_conf) ();
507 thread_index = vlib_get_thread_index ();
508 if (pool_elts (om->evt_pool[thread_index]) > 0)
510 openssl_async_polling ();
511 tls_resume_from_crypto (thread_index);
517 VLIB_INIT_FUNCTION (tls_async_init);
520 VLIB_REGISTER_NODE (tls_async_process_node,static) = {
521 .function = tls_async_process,
522 .type = VLIB_NODE_TYPE_INPUT,
523 .name = "tls-async-process",
530 * fd.io coding-style-patch-verification: ON
533 * eval: (c-set-style "gnu")