2 * Copyright (c) 2018 Intel and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
15 #include <vnet/vnet.h>
16 #include <vnet/api_errno.h>
17 #include <vlib/node_funcs.h>
18 #include <openssl/engine.h>
19 #include <tlsopenssl/tls_openssl.h>
21 #define SSL_ASYNC_INFLIGHT 1
22 #define SSL_ASYNC_READY 2
23 #define SSL_ASYNC_REENTER 3
24 #define MAX_VECTOR_ASYNC 256
26 typedef struct openssl_tls_callback_arg_
30 } openssl_tls_callback_arg_t;
32 typedef struct openssl_event_
38 openssl_resume_handler *handler;
39 openssl_tls_callback_arg_t cb_args;
40 #define thread_idx cb_args.thread_index
41 #define event_idx cb_args.event_index
45 typedef struct openssl_async_queue_
49 } openssl_async_queue_t;
51 typedef struct openssl_async_
53 openssl_evt_t ***evt_pool;
54 openssl_async_queue_t *queue;
55 void (*polling) (void);
63 void qat_polling_config ();
64 void dasync_polling ();
69 void (*polling) (void);
70 void (*pre_init) (void);
71 void (*thread_init) (void *);
74 void qat_init_thread (void *arg);
76 struct engine_polling engine_list[] = {
77 {"qat", qat_polling, qat_pre_init, qat_init_thread},
78 {"dasync", dasync_polling, NULL, NULL}
81 openssl_async_t openssl_async_main;
82 static vlib_node_registration_t tls_async_process_node;
84 /* to avoid build warning */
85 void session_send_rpc_evt_to_thread (u32 thread_index, void *fp,
89 evt_pool_init (vlib_main_t * vm)
91 vlib_thread_main_t *vtm = vlib_get_thread_main ();
92 openssl_async_t *om = &openssl_async_main;
95 num_threads = 1 /* main thread */ + vtm->n_threads;
97 TLS_DBG (2, "Totally there is %d thread\n", num_threads);
99 vec_validate (om->evt_pool, num_threads - 1);
100 vec_validate (om->queue, num_threads - 1);
102 om->start_polling = 0;
105 for (i = 0; i < num_threads; i++)
107 om->queue[i].evt_run_head = -1;
108 om->queue[i].evt_run_tail = -1;
116 openssl_engine_register (char *engine_name, char *algorithm, int async)
118 int i, registered = -1;
119 openssl_async_t *om = &openssl_async_main;
123 for (i = 0; i < ARRAY_LEN (engine_list); i++)
125 if (!strcmp (engine_list[i].engine, engine_name))
127 om->polling = engine_list[i].polling;
133 clib_error ("engine %s is not regisered in VPP", engine_name);
137 ENGINE_load_builtin_engines ();
138 ENGINE_load_dynamic ();
139 engine = ENGINE_by_id (engine_name);
143 clib_warning ("Failed to find engine ENGINE_by_id %s", engine_name);
149 p = engine_list[registered].pre_init;
155 if (!ENGINE_set_default_string (engine, algorithm))
157 clib_warning ("Failed to set engine %s algorithm %s\n",
158 engine_name, algorithm);
164 if (!ENGINE_set_default (engine, ENGINE_METHOD_ALL))
166 clib_warning ("Failed to set engine %s to all algorithm",
174 openssl_async_node_enable_disable (1);
177 for (i = 0; i < vlib_num_workers (); i++)
179 if (engine_list[registered].thread_init)
180 session_send_rpc_evt_to_thread (i + 1,
181 engine_list[registered].thread_init,
185 om->start_polling = 1;
191 static openssl_evt_t *
192 openssl_evt_get (u32 evt_index)
196 pool_elt_at_index (openssl_async_main.evt_pool[vlib_get_thread_index ()],
201 static openssl_evt_t *
202 openssl_evt_get_w_thread (int evt_index, u8 thread_index)
207 pool_elt_at_index (openssl_async_main.evt_pool[thread_index], evt_index);
212 openssl_evt_free (int event_index, u8 thread_index)
214 openssl_async_t *om = &openssl_async_main;
217 pool_put_index (om->evt_pool[thread_index], event_index);
223 openssl_evt_alloc (void)
225 u8 thread_index = vlib_get_thread_index ();
226 openssl_async_t *tm = &openssl_async_main;
229 pool_get (tm->evt_pool[thread_index], evt);
231 *evt = clib_mem_alloc (sizeof (openssl_evt_t));
233 clib_memset (*evt, 0, sizeof (openssl_evt_t));
234 (*evt)->event_idx = evt - tm->evt_pool[thread_index];
235 return ((*evt)->event_idx);
239 /* In most cases, tls_async_openssl_callback is called by HW to make event active
240 * When EAGAIN received, VPP will call this callback to retry
243 tls_async_openssl_callback (SSL * s, void *cb_arg)
245 openssl_evt_t *event, *event_tail;
246 openssl_async_t *om = &openssl_async_main;
247 openssl_tls_callback_arg_t *args = (openssl_tls_callback_arg_t *) cb_arg;
248 int thread_index = args->thread_index;
249 int event_index = args->event_index;
250 int *evt_run_tail = &om->queue[thread_index].evt_run_tail;
251 int *evt_run_head = &om->queue[thread_index].evt_run_head;
253 TLS_DBG (2, "Set event %d to run\n", event_index);
254 event = openssl_evt_get_w_thread (event_index, thread_index);
256 /* Happend when a recursive case, especially in SW simulation */
257 if (PREDICT_FALSE (event->status == SSL_ASYNC_READY))
259 event->status = SSL_ASYNC_REENTER;
262 event->status = SSL_ASYNC_READY;
265 if (*evt_run_tail >= 0)
267 event_tail = openssl_evt_get_w_thread (*evt_run_tail, thread_index);
268 event_tail->next = event_index;
270 *evt_run_tail = event_index;
271 if (*evt_run_head < 0)
273 *evt_run_head = event_index;
280 vpp_tls_async_init_event (tls_ctx_t * ctx,
281 openssl_resume_handler * handler,
285 openssl_evt_t *event;
286 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
287 u32 thread_id = ctx->c_thread_index;
289 eidx = openssl_evt_alloc ();
290 event = openssl_evt_get (eidx);
291 event->ctx_index = oc->openssl_ctx_index;
292 event->event_idx = eidx;
293 event->thread_idx = thread_id;
294 event->handler = handler;
295 event->session_index = session->session_index;
297 ctx->evt_index = eidx;
298 #ifdef HAVE_OPENSSL_ASYNC
299 SSL_set_async_callback_arg (oc->ssl, &event->cb_args);
306 vpp_openssl_is_inflight (tls_ctx_t * ctx)
309 openssl_evt_t *event;
310 eidx = ctx->evt_index;
311 event = openssl_evt_get (eidx);
313 if (event->status == SSL_ASYNC_INFLIGHT)
319 vpp_tls_async_update_event (tls_ctx_t * ctx, int eagain)
322 openssl_evt_t *event;
324 eidx = ctx->evt_index;
325 event = openssl_evt_get (eidx);
326 event->status = SSL_ASYNC_INFLIGHT;
328 return tls_async_openssl_callback (0, &event->cb_args);
334 event_handler (void *tls_async)
336 openssl_resume_handler *handler;
337 openssl_evt_t *event;
342 event = (openssl_evt_t *) tls_async;
343 thread_index = event->thread_idx;
344 ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index);
345 handler = event->handler;
346 session = session_get (event->session_index, thread_index);
350 (*handler) (ctx, session);
356 /* engine specific code to polling the response ring */
360 /* dasync is a fake async device, and could not be polled.
361 * We have added code in the dasync engine to triggered the callback already,
362 * so nothing can be done here
369 openssl_async_t *om = &openssl_async_main;
371 ENGINE_ctrl_cmd (om->engine, "ENABLE_EXTERNAL_POLLING", 0, NULL, NULL, 0);
374 /* Below code is spefic to QAT engine, and other vendors can refer to this code to enable a new engine */
376 qat_init_thread (void *arg)
378 openssl_async_t *om = &openssl_async_main;
379 int thread_index = *(int *) arg;
381 ENGINE_ctrl_cmd (om->engine, "SET_INSTANCE_FOR_THREAD", thread_index,
384 TLS_DBG (2, "set thread %d and instance %d mapping\n", thread_index,
392 openssl_async_t *om = &openssl_async_main;
395 if (om->start_polling)
397 ENGINE_ctrl_cmd (om->engine, "POLL", 0, &poll_status, NULL, 0);
402 openssl_async_polling ()
404 openssl_async_t *om = &openssl_async_main;
412 openssl_async_node_enable_disable (u8 is_en)
414 u8 state = is_en ? VLIB_NODE_STATE_POLLING : VLIB_NODE_STATE_DISABLED;
415 vlib_thread_main_t *vtm = vlib_get_thread_main ();
416 u8 have_workers = vtm->n_threads != 0;
419 foreach_vlib_main (({
420 if (have_workers && ii != 0)
422 vlib_node_set_state (this_vlib_main, tls_async_process_node.index,
430 tls_async_do_job (int eidx, u32 thread_index)
433 openssl_evt_t *event;
435 /* do the real job */
436 event = openssl_evt_get_w_thread (eidx, thread_index);
437 ctx = openssl_ctx_get_w_thread (event->ctx_index, thread_index);
442 session_send_rpc_evt_to_thread (thread_index, event_handler, event);
448 tls_resume_from_crypto (int thread_index)
452 openssl_async_t *om = &openssl_async_main;
453 openssl_evt_t *event;
454 int *evt_run_head = &om->queue[thread_index].evt_run_head;
455 int *evt_run_tail = &om->queue[thread_index].evt_run_tail;
457 if (*evt_run_head < 0)
460 for (i = 0; i < MAX_VECTOR_ASYNC; i++)
462 if (*evt_run_head >= 0)
464 event = openssl_evt_get_w_thread (*evt_run_head, thread_index);
465 tls_async_do_job (*evt_run_head, thread_index);
466 if (PREDICT_FALSE (event->status == SSL_ASYNC_REENTER))
468 /* recusive event triggered */
469 event->status = SSL_ASYNC_READY;
474 *evt_run_head = event->next;
488 static clib_error_t *
489 tls_async_init (vlib_main_t * vm)
496 tls_async_process (vlib_main_t * vm, vlib_node_runtime_t * rt,
500 openssl_async_t *om = &openssl_async_main;
502 thread_index = vlib_get_thread_index ();
503 if (pool_elts (om->evt_pool[thread_index]) > 0)
505 openssl_async_polling ();
506 tls_resume_from_crypto (thread_index);
512 VLIB_INIT_FUNCTION (tls_async_init);
515 VLIB_REGISTER_NODE (tls_async_process_node,static) = {
516 .function = tls_async_process,
517 .type = VLIB_NODE_TYPE_INPUT,
518 .name = "tls-async-process",
519 .state = VLIB_NODE_STATE_DISABLED,
525 * fd.io coding-style-patch-verification: ON
528 * eval: (c-set-style "gnu")