2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <openssl/ssl.h>
17 #include <openssl/conf.h>
18 #include <openssl/err.h>
19 #ifdef HAVE_OPENSSL_ASYNC
20 #include <openssl/async.h>
23 #include <vnet/plugin/plugin.h>
24 #include <vpp/app/version.h>
25 #include <vnet/tls/tls.h>
27 #include <tlsopenssl/tls_openssl.h>
29 #define MAX_CRYPTO_LEN 16
31 static openssl_main_t openssl_main;
33 openssl_ctx_alloc (void)
35 u8 thread_index = vlib_get_thread_index ();
36 openssl_main_t *tm = &openssl_main;
39 pool_get (tm->ctx_pool[thread_index], ctx);
41 *ctx = clib_mem_alloc (sizeof (openssl_ctx_t));
43 memset (*ctx, 0, sizeof (openssl_ctx_t));
44 (*ctx)->ctx.c_thread_index = thread_index;
45 (*ctx)->ctx.tls_ctx_engine = TLS_ENGINE_OPENSSL;
46 (*ctx)->ctx.app_session_handle = SESSION_INVALID_HANDLE;
47 (*ctx)->openssl_ctx_index = ctx - tm->ctx_pool[thread_index];
48 return ((*ctx)->openssl_ctx_index);
52 openssl_ctx_free (tls_ctx_t * ctx)
54 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
56 if (SSL_is_init_finished (oc->ssl) && !ctx->is_passive_close)
57 SSL_shutdown (oc->ssl);
59 if (SSL_is_server (oc->ssl))
61 X509_free (oc->srvcert);
62 EVP_PKEY_free (oc->pkey);
66 pool_put_index (openssl_main.ctx_pool[ctx->c_thread_index],
67 oc->openssl_ctx_index);
71 openssl_ctx_get (u32 ctx_index)
74 ctx = pool_elt_at_index (openssl_main.ctx_pool[vlib_get_thread_index ()],
80 openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index)
83 ctx = pool_elt_at_index (openssl_main.ctx_pool[thread_index], ctx_index);
88 openssl_try_handshake_read (openssl_ctx_t * oc,
89 stream_session_t * tls_session)
95 f = tls_session->server_rx_fifo;
96 deq_max = svm_fifo_max_dequeue (f);
100 deq_now = clib_min (svm_fifo_max_read_chunk (f), deq_max);
101 wrote = BIO_write (oc->wbio, svm_fifo_head (f), deq_now);
105 svm_fifo_dequeue_drop (f, wrote);
108 deq_now = clib_min (svm_fifo_max_read_chunk (f), deq_max - wrote);
109 rv = BIO_write (oc->wbio, svm_fifo_head (f), deq_now);
112 svm_fifo_dequeue_drop (f, rv);
120 openssl_try_handshake_write (openssl_ctx_t * oc,
121 stream_session_t * tls_session)
123 u32 enq_max, deq_now;
127 if (BIO_ctrl_pending (oc->rbio) <= 0)
130 f = tls_session->server_tx_fifo;
131 enq_max = svm_fifo_max_enqueue (f);
135 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max);
136 read = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
140 svm_fifo_enqueue_nocopy (f, read);
141 tls_add_vpp_q_evt (f, FIFO_EVENT_APP_TX);
145 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max - read);
146 rv = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
149 svm_fifo_enqueue_nocopy (f, rv);
157 #ifdef HAVE_OPENSSL_ASYNC
159 vpp_ssl_async_process_event (tls_ctx_t * ctx,
160 openssl_resume_handler * handler)
162 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
163 openssl_tls_callback_t *engine_cb;
165 engine_cb = vpp_add_async_pending_event (ctx, handler);
168 SSL_set_async_callback (oc->ssl, (void *) engine_cb->callback,
169 (void *) engine_cb->arg);
170 TLS_DBG (2, "set callback to engine %p\n", engine_cb->callback);
176 /* Due to engine busy stat, VPP need to retry later */
178 vpp_ssl_async_retry_func (tls_ctx_t * ctx, openssl_resume_handler * handler)
180 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
182 if (vpp_add_async_run_event (ctx, handler))
184 SSL_set_async_estatus (oc->ssl, 0);
193 openssl_ctx_handshake_rx (tls_ctx_t * ctx, stream_session_t * tls_session)
195 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
197 #ifdef HAVE_OPENSSL_ASYNC
199 openssl_resume_handler *myself;
202 while (SSL_in_init (oc->ssl))
208 else if (!openssl_try_handshake_read (oc, tls_session))
213 rv = SSL_do_handshake (oc->ssl);
214 err = SSL_get_error (oc->ssl, rv);
215 openssl_try_handshake_write (oc, tls_session);
216 #ifdef HAVE_OPENSSL_ASYNC
217 myself = openssl_ctx_handshake_rx;
218 if (SSL_get_async_estatus (oc->ssl, &estatus)
219 && (estatus == ENGINE_STATUS_RETRY))
221 vpp_ssl_async_retry_func (ctx, myself);
223 else if (err == SSL_ERROR_WANT_ASYNC)
225 vpp_ssl_async_process_event (ctx, myself);
229 if (err != SSL_ERROR_WANT_WRITE)
231 if (err == SSL_ERROR_SSL)
234 ERR_error_string (ERR_get_error (), buf);
235 clib_warning ("Err: %s", buf);
240 TLS_DBG (2, "tls state for %u is %s", oc->openssl_ctx_index,
241 SSL_state_string_long (oc->ssl));
243 if (SSL_in_init (oc->ssl))
249 if (!SSL_is_server (oc->ssl))
252 * Verify server certificate
254 if ((rv = SSL_get_verify_result (oc->ssl)) != X509_V_OK)
256 TLS_DBG (1, " failed verify: %s\n",
257 X509_verify_cert_error_string (rv));
260 * Presence of hostname enforces strict certificate verification
262 if (ctx->srv_hostname)
264 tls_notify_app_connected (ctx, /* is failed */ 0);
268 tls_notify_app_connected (ctx, /* is failed */ 0);
272 tls_notify_app_accept (ctx);
275 TLS_DBG (1, "Handshake for %u complete. TLS cipher is %s",
276 oc->openssl_ctx_index, SSL_get_cipher (oc->ssl));
281 openssl_ctx_write (tls_ctx_t * ctx, stream_session_t * app_session)
283 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
284 int wrote = 0, rv, read, max_buf = 100 * TLS_CHUNK_SIZE, max_space;
285 u32 enq_max, deq_max, deq_now, to_write;
286 stream_session_t *tls_session;
289 f = app_session->server_tx_fifo;
290 deq_max = svm_fifo_max_dequeue (f);
294 max_space = max_buf - BIO_ctrl_pending (oc->rbio);
295 max_space = (max_space < 0) ? 0 : max_space;
296 deq_now = clib_min (deq_max, (u32) max_space);
297 to_write = clib_min (svm_fifo_max_read_chunk (f), deq_now);
298 wrote = SSL_write (oc->ssl, svm_fifo_head (f), to_write);
301 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
304 svm_fifo_dequeue_drop (app_session->server_tx_fifo, wrote);
307 to_write = clib_min (svm_fifo_max_read_chunk (f), deq_now - wrote);
308 rv = SSL_write (oc->ssl, svm_fifo_head (f), to_write);
311 svm_fifo_dequeue_drop (app_session->server_tx_fifo, rv);
316 if (deq_now < deq_max)
317 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
321 if (BIO_ctrl_pending (oc->rbio) <= 0)
324 tls_session = session_get_from_handle (ctx->tls_session_handle);
325 f = tls_session->server_tx_fifo;
326 enq_max = svm_fifo_max_enqueue (f);
329 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
333 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max);
334 read = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
337 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
341 svm_fifo_enqueue_nocopy (f, read);
342 tls_add_vpp_q_evt (f, FIFO_EVENT_APP_TX);
344 if (read < enq_max && BIO_ctrl_pending (oc->rbio) > 0)
346 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max - read);
347 read = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
349 svm_fifo_enqueue_nocopy (f, read);
352 if (BIO_ctrl_pending (oc->rbio) > 0)
353 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
359 openssl_ctx_read (tls_ctx_t * ctx, stream_session_t * tls_session)
361 int read, wrote = 0, max_space, max_buf = 100 * TLS_CHUNK_SIZE, rv;
362 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
363 u32 deq_max, enq_max, deq_now, to_read;
364 stream_session_t *app_session;
367 if (PREDICT_FALSE (SSL_in_init (oc->ssl)))
369 openssl_ctx_handshake_rx (ctx, tls_session);
373 f = tls_session->server_rx_fifo;
374 deq_max = svm_fifo_max_dequeue (f);
375 max_space = max_buf - BIO_ctrl_pending (oc->wbio);
376 max_space = max_space < 0 ? 0 : max_space;
377 deq_now = clib_min (deq_max, max_space);
381 to_read = clib_min (svm_fifo_max_read_chunk (f), deq_now);
382 wrote = BIO_write (oc->wbio, svm_fifo_head (f), to_read);
385 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
388 svm_fifo_dequeue_drop (f, wrote);
391 to_read = clib_min (svm_fifo_max_read_chunk (f), deq_now - wrote);
392 rv = BIO_write (oc->wbio, svm_fifo_head (f), to_read);
395 svm_fifo_dequeue_drop (f, rv);
399 if (svm_fifo_max_dequeue (f))
400 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
404 if (BIO_ctrl_pending (oc->wbio) <= 0)
407 app_session = session_get_from_handle (ctx->app_session_handle);
408 f = app_session->server_rx_fifo;
409 enq_max = svm_fifo_max_enqueue (f);
412 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
416 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max);
417 read = SSL_read (oc->ssl, svm_fifo_tail (f), deq_now);
420 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
423 svm_fifo_enqueue_nocopy (f, read);
424 if (read < enq_max && BIO_ctrl_pending (oc->wbio) > 0)
426 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max - read);
427 read = SSL_read (oc->ssl, svm_fifo_tail (f), deq_now);
429 svm_fifo_enqueue_nocopy (f, read);
432 tls_notify_app_enqueue (ctx, app_session);
433 if (BIO_ctrl_pending (oc->wbio) > 0)
434 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
440 openssl_ctx_init_client (tls_ctx_t * ctx)
442 char *ciphers = "ALL:!ADH:!LOW:!EXP:!MD5:!RC4-SHA:!DES-CBC3-SHA:@STRENGTH";
443 long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION;
444 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
445 openssl_main_t *om = &openssl_main;
446 stream_session_t *tls_session;
447 const SSL_METHOD *method;
449 #ifdef HAVE_OPENSSL_ASYNC
450 openssl_resume_handler *handler;
453 method = SSLv23_client_method ();
456 TLS_DBG (1, "SSLv23_method returned null");
460 oc->ssl_ctx = SSL_CTX_new (method);
461 if (oc->ssl_ctx == NULL)
463 TLS_DBG (1, "SSL_CTX_new returned null");
467 SSL_CTX_set_ecdh_auto (oc->ssl_ctx, 1);
468 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
469 #ifdef HAVE_OPENSSL_ASYNC
471 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ASYNC);
473 rv = SSL_CTX_set_cipher_list (oc->ssl_ctx, (const char *) ciphers);
476 TLS_DBG (1, "Couldn't set cipher");
480 SSL_CTX_set_options (oc->ssl_ctx, flags);
481 SSL_CTX_set_cert_store (oc->ssl_ctx, om->cert_store);
483 oc->ssl = SSL_new (oc->ssl_ctx);
486 TLS_DBG (1, "Couldn't initialize ssl struct");
490 oc->rbio = BIO_new (BIO_s_mem ());
491 oc->wbio = BIO_new (BIO_s_mem ());
493 BIO_set_mem_eof_return (oc->rbio, -1);
494 BIO_set_mem_eof_return (oc->wbio, -1);
496 SSL_set_bio (oc->ssl, oc->wbio, oc->rbio);
497 SSL_set_connect_state (oc->ssl);
499 rv = SSL_set_tlsext_host_name (oc->ssl, ctx->srv_hostname);
502 TLS_DBG (1, "Couldn't set hostname");
507 * 2. Do the first steps in the handshake.
509 TLS_DBG (1, "Initiating handshake for [%u]%u", ctx->c_thread_index,
510 oc->openssl_ctx_index);
512 tls_session = session_get_from_handle (ctx->tls_session_handle);
515 rv = SSL_do_handshake (oc->ssl);
516 err = SSL_get_error (oc->ssl, rv);
517 openssl_try_handshake_write (oc, tls_session);
518 #ifdef HAVE_OPENSSL_ASYNC
519 if (err == SSL_ERROR_WANT_ASYNC)
521 handler = (openssl_resume_handler *) openssl_ctx_handshake_rx;
522 vpp_ssl_async_process_event (ctx, handler);
526 if (err != SSL_ERROR_WANT_WRITE)
530 TLS_DBG (2, "tls state for [%u]%u is su", ctx->c_thread_index,
531 oc->openssl_ctx_index, SSL_state_string_long (oc->ssl));
536 openssl_ctx_init_server (tls_ctx_t * ctx)
538 char *ciphers = "ALL:!ADH:!LOW:!EXP:!MD5:!RC4-SHA:!DES-CBC3-SHA:@STRENGTH";
539 long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION;
540 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
541 stream_session_t *tls_session;
542 const SSL_METHOD *method;
546 #ifdef HAVE_OPENSSL_ASYNC
547 openssl_main_t *om = &openssl_main;
548 openssl_resume_handler *handler;
551 app = application_get (ctx->parent_app_index);
552 if (!app->tls_cert || !app->tls_key)
554 TLS_DBG (1, "tls cert and/or key not configured %d",
555 ctx->parent_app_index);
559 method = SSLv23_method ();
560 oc->ssl_ctx = SSL_CTX_new (method);
563 clib_warning ("Unable to create SSL context");
567 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
568 #ifdef HAVE_OPENSSL_ASYNC
570 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ASYNC);
572 SSL_CTX_set_options (oc->ssl_ctx, flags);
573 SSL_CTX_set_ecdh_auto (oc->ssl_ctx, 1);
575 rv = SSL_CTX_set_cipher_list (oc->ssl_ctx, (const char *) ciphers);
578 TLS_DBG (1, "Couldn't set cipher");
583 * Set the key and cert
585 cert_bio = BIO_new (BIO_s_mem ());
586 BIO_write (cert_bio, app->tls_cert, vec_len (app->tls_cert));
587 oc->srvcert = PEM_read_bio_X509 (cert_bio, NULL, NULL, NULL);
590 clib_warning ("unable to parse certificate");
593 SSL_CTX_use_certificate (oc->ssl_ctx, oc->srvcert);
597 cert_bio = BIO_new (BIO_s_mem ());
598 BIO_write (cert_bio, app->tls_key, vec_len (app->tls_key));
599 oc->pkey = PEM_read_bio_PrivateKey (cert_bio, NULL, NULL, NULL);
602 clib_warning ("unable to parse pkey");
605 SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
607 SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
610 /* Start a new connection */
612 oc->ssl = SSL_new (oc->ssl_ctx);
615 TLS_DBG (1, "Couldn't initialize ssl struct");
619 oc->rbio = BIO_new (BIO_s_mem ());
620 oc->wbio = BIO_new (BIO_s_mem ());
622 BIO_set_mem_eof_return (oc->rbio, -1);
623 BIO_set_mem_eof_return (oc->wbio, -1);
625 SSL_set_bio (oc->ssl, oc->wbio, oc->rbio);
626 SSL_set_accept_state (oc->ssl);
628 TLS_DBG (1, "Initiating handshake for [%u]%u", ctx->c_thread_index,
629 oc->openssl_ctx_index);
631 tls_session = session_get_from_handle (ctx->tls_session_handle);
634 rv = SSL_do_handshake (oc->ssl);
635 err = SSL_get_error (oc->ssl, rv);
636 openssl_try_handshake_write (oc, tls_session);
637 #ifdef HAVE_OPENSSL_ASYNC
638 if (err == SSL_ERROR_WANT_ASYNC)
640 handler = (openssl_resume_handler *) openssl_ctx_handshake_rx;
641 vpp_ssl_async_process_event (ctx, handler);
645 if (err != SSL_ERROR_WANT_WRITE)
649 TLS_DBG (2, "tls state for [%u]%u is su", ctx->c_thread_index,
650 oc->openssl_ctx_index, SSL_state_string_long (oc->ssl));
655 openssl_handshake_is_over (tls_ctx_t * ctx)
657 openssl_ctx_t *mc = (openssl_ctx_t *) ctx;
660 return SSL_is_init_finished (mc->ssl);
663 const static tls_engine_vft_t openssl_engine = {
664 .ctx_alloc = openssl_ctx_alloc,
665 .ctx_free = openssl_ctx_free,
666 .ctx_get = openssl_ctx_get,
667 .ctx_get_w_thread = openssl_ctx_get_w_thread,
668 .ctx_init_server = openssl_ctx_init_server,
669 .ctx_init_client = openssl_ctx_init_client,
670 .ctx_write = openssl_ctx_write,
671 .ctx_read = openssl_ctx_read,
672 .ctx_handshake_is_over = openssl_handshake_is_over,
676 tls_init_ca_chain (void)
678 openssl_main_t *om = &openssl_main;
679 tls_main_t *tm = vnet_tls_get_main ();
684 if (access (tm->ca_cert_path, F_OK | R_OK) == -1)
686 clib_warning ("Could not initialize TLS CA certificates");
690 if (!(om->cert_store = X509_STORE_new ()))
692 clib_warning ("failed to create cert store");
696 rv = X509_STORE_load_locations (om->cert_store, tm->ca_cert_path, 0);
699 clib_warning ("failed to load ca certificate");
702 if (tm->use_test_cert_in_ca)
704 cert_bio = BIO_new (BIO_s_mem ());
705 BIO_write (cert_bio, test_srv_crt_rsa, test_srv_crt_rsa_len);
706 testcert = PEM_read_bio_X509 (cert_bio, NULL, NULL, NULL);
709 clib_warning ("unable to parse certificate");
712 X509_STORE_add_cert (om->cert_store, testcert);
715 return (rv < 0 ? -1 : 0);
718 static clib_error_t *
719 tls_openssl_init (vlib_main_t * vm)
721 vlib_thread_main_t *vtm = vlib_get_thread_main ();
722 openssl_main_t *om = &openssl_main;
726 num_threads = 1 /* main thread */ + vtm->n_threads;
728 if ((error = vlib_call_init_function (vm, tls_init)))
732 SSL_load_error_strings ();
734 if (tls_init_ca_chain ())
736 clib_warning ("failed to initialize TLS CA chain");
740 vec_validate (om->ctx_pool, num_threads - 1);
742 tls_register_engine (&openssl_engine, TLS_ENGINE_OPENSSL);
749 #ifdef HAVE_OPENSSL_ASYNC
750 static clib_error_t *
751 tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input,
752 vlib_cli_command_t * cmd)
754 openssl_main_t *om = &openssl_main;
755 char *engine_name = NULL;
756 char *engine_alg = NULL;
757 u8 engine_name_set = 0;
760 /* By present, it is not allowed to configure engine again after running */
763 clib_warning ("engine has started!\n");
764 return clib_error_return
765 (0, "engine has started, and no config is accepted");
768 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
770 if (unformat (input, "engine %s", &engine_name))
774 else if (unformat (input, "async"))
777 openssl_async_node_enable_disable (1);
779 else if (unformat (input, "alg %s", &engine_alg))
781 for (i = 0; i < strnlen (engine_alg, MAX_CRYPTO_LEN); i++)
782 engine_alg[i] = toupper (engine_alg[i]);
785 return clib_error_return (0, "failed: unknown input `%U'",
786 format_unformat_error, input);
789 /* reset parameters if engine is not configured */
790 if (!engine_name_set)
792 clib_warning ("No engine provided! \n");
797 if (!openssl_engine_register (engine_name, engine_alg))
799 return clib_error_return (0, "failed to register %s polling",
808 VLIB_CLI_COMMAND (tls_openssl_set_command, static) =
810 .path = "tls openssl set",
811 .short_help = "tls openssl set [engine <engine name>] [alg [algorithm] [async]",
812 .function = tls_openssl_set_command_fn,
818 VLIB_INIT_FUNCTION (tls_openssl_init);
821 VLIB_PLUGIN_REGISTER () = {
822 .version = VPP_BUILD_VER,
823 .description = "openssl based TLS Engine",
828 * fd.io coding-style-patch-verification: ON
831 * eval: (c-set-style "gnu")