2 * Copyright (c) 2018 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 #include <openssl/ssl.h>
17 #include <openssl/conf.h>
18 #include <openssl/err.h>
19 #ifdef HAVE_OPENSSL_ASYNC
20 #include <openssl/async.h>
23 #include <vnet/plugin/plugin.h>
24 #include <vpp/app/version.h>
25 #include <vnet/tls/tls.h>
27 #include <tlsopenssl/tls_openssl.h>
29 #define MAX_CRYPTO_LEN 16
31 static openssl_main_t openssl_main;
33 openssl_ctx_alloc (void)
35 u8 thread_index = vlib_get_thread_index ();
36 openssl_main_t *tm = &openssl_main;
39 pool_get (tm->ctx_pool[thread_index], ctx);
41 *ctx = clib_mem_alloc (sizeof (openssl_ctx_t));
43 memset (*ctx, 0, sizeof (openssl_ctx_t));
44 (*ctx)->ctx.c_thread_index = thread_index;
45 (*ctx)->ctx.tls_ctx_engine = TLS_ENGINE_OPENSSL;
46 (*ctx)->ctx.app_session_handle = SESSION_INVALID_HANDLE;
47 (*ctx)->openssl_ctx_index = ctx - tm->ctx_pool[thread_index];
48 return ((*ctx)->openssl_ctx_index);
52 openssl_ctx_free (tls_ctx_t * ctx)
54 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
56 if (SSL_is_init_finished (oc->ssl) && !ctx->is_passive_close)
57 SSL_shutdown (oc->ssl);
59 if (SSL_is_server (oc->ssl))
61 X509_free (oc->srvcert);
62 EVP_PKEY_free (oc->pkey);
66 pool_put_index (openssl_main.ctx_pool[ctx->c_thread_index],
67 oc->openssl_ctx_index);
71 openssl_ctx_get (u32 ctx_index)
74 ctx = pool_elt_at_index (openssl_main.ctx_pool[vlib_get_thread_index ()],
80 openssl_ctx_get_w_thread (u32 ctx_index, u8 thread_index)
83 ctx = pool_elt_at_index (openssl_main.ctx_pool[thread_index], ctx_index);
88 openssl_try_handshake_read (openssl_ctx_t * oc,
89 stream_session_t * tls_session)
95 f = tls_session->server_rx_fifo;
96 deq_max = svm_fifo_max_dequeue (f);
100 deq_now = clib_min (svm_fifo_max_read_chunk (f), deq_max);
101 wrote = BIO_write (oc->wbio, svm_fifo_head (f), deq_now);
105 svm_fifo_dequeue_drop (f, wrote);
108 deq_now = clib_min (svm_fifo_max_read_chunk (f), deq_max - wrote);
109 rv = BIO_write (oc->wbio, svm_fifo_head (f), deq_now);
112 svm_fifo_dequeue_drop (f, rv);
120 openssl_try_handshake_write (openssl_ctx_t * oc,
121 stream_session_t * tls_session)
123 u32 enq_max, deq_now;
127 if (BIO_ctrl_pending (oc->rbio) <= 0)
130 f = tls_session->server_tx_fifo;
131 enq_max = svm_fifo_max_enqueue (f);
135 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max);
136 read = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
140 svm_fifo_enqueue_nocopy (f, read);
141 tls_add_vpp_q_evt (f, FIFO_EVENT_APP_TX);
145 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max - read);
146 rv = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
149 svm_fifo_enqueue_nocopy (f, rv);
157 #ifdef HAVE_OPENSSL_ASYNC
159 vpp_ssl_async_process_event (tls_ctx_t * ctx,
160 openssl_resume_handler * handler)
162 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
163 openssl_tls_callback_t *engine_cb;
165 engine_cb = vpp_add_async_pending_event (ctx, handler);
168 SSL_set_async_callback (oc->ssl, (void *) engine_cb->callback,
169 (void *) engine_cb->arg);
170 TLS_DBG ("set callback to engine %p\n", engine_cb->callback);
172 /* associated fd with context for return */
173 TLS_DBG ("completed assoicated fd with tls session\n");
180 openssl_ctx_handshake_rx (tls_ctx_t * ctx, stream_session_t * tls_session)
182 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
184 #ifdef HAVE_OPENSSL_ASYNC
185 openssl_resume_handler *myself;
188 while (SSL_in_init (oc->ssl))
194 else if (!openssl_try_handshake_read (oc, tls_session))
199 rv = SSL_do_handshake (oc->ssl);
200 err = SSL_get_error (oc->ssl, rv);
201 openssl_try_handshake_write (oc, tls_session);
202 #ifdef HAVE_OPENSSL_ASYNC
203 if (err == SSL_ERROR_WANT_ASYNC)
205 myself = openssl_ctx_handshake_rx;
206 vpp_ssl_async_process_event (ctx, myself);
210 if (err != SSL_ERROR_WANT_WRITE)
212 if (err == SSL_ERROR_SSL)
215 ERR_error_string (ERR_get_error (), buf);
216 clib_warning ("Err: %s", buf);
221 TLS_DBG (2, "tls state for %u is %s", oc->openssl_ctx_index,
222 SSL_state_string_long (oc->ssl));
224 if (SSL_in_init (oc->ssl))
230 if (!SSL_is_server (oc->ssl))
233 * Verify server certificate
235 if ((rv = SSL_get_verify_result (oc->ssl)) != X509_V_OK)
237 TLS_DBG (1, " failed verify: %s\n",
238 X509_verify_cert_error_string (rv));
241 * Presence of hostname enforces strict certificate verification
243 if (ctx->srv_hostname)
245 tls_notify_app_connected (ctx, /* is failed */ 0);
249 tls_notify_app_connected (ctx, /* is failed */ 0);
253 tls_notify_app_accept (ctx);
256 TLS_DBG (1, "Handshake for %u complete. TLS cipher is %s",
257 oc->openssl_ctx_index, SSL_get_cipher (oc->ssl));
262 openssl_ctx_write (tls_ctx_t * ctx, stream_session_t * app_session)
264 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
265 int wrote = 0, rv, read, max_buf = 100 * TLS_CHUNK_SIZE, max_space;
266 u32 enq_max, deq_max, deq_now, to_write;
267 stream_session_t *tls_session;
270 f = app_session->server_tx_fifo;
271 deq_max = svm_fifo_max_dequeue (f);
275 max_space = max_buf - BIO_ctrl_pending (oc->rbio);
276 max_space = (max_space < 0) ? 0 : max_space;
277 deq_now = clib_min (deq_max, (u32) max_space);
278 to_write = clib_min (svm_fifo_max_read_chunk (f), deq_now);
279 wrote = SSL_write (oc->ssl, svm_fifo_head (f), to_write);
282 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
285 svm_fifo_dequeue_drop (app_session->server_tx_fifo, wrote);
288 to_write = clib_min (svm_fifo_max_read_chunk (f), deq_now - wrote);
289 rv = SSL_write (oc->ssl, svm_fifo_head (f), to_write);
292 svm_fifo_dequeue_drop (app_session->server_tx_fifo, rv);
297 if (deq_now < deq_max)
298 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
302 if (BIO_ctrl_pending (oc->rbio) <= 0)
305 tls_session = session_get_from_handle (ctx->tls_session_handle);
306 f = tls_session->server_tx_fifo;
307 enq_max = svm_fifo_max_enqueue (f);
310 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
314 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max);
315 read = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
318 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
322 svm_fifo_enqueue_nocopy (f, read);
323 tls_add_vpp_q_evt (f, FIFO_EVENT_APP_TX);
325 if (read < enq_max && BIO_ctrl_pending (oc->rbio) > 0)
327 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max - read);
328 read = BIO_read (oc->rbio, svm_fifo_tail (f), deq_now);
330 svm_fifo_enqueue_nocopy (f, read);
333 if (BIO_ctrl_pending (oc->rbio) > 0)
334 tls_add_vpp_q_evt (app_session->server_tx_fifo, FIFO_EVENT_APP_TX);
340 openssl_ctx_read (tls_ctx_t * ctx, stream_session_t * tls_session)
342 int read, wrote = 0, max_space, max_buf = 100 * TLS_CHUNK_SIZE, rv;
343 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
344 u32 deq_max, enq_max, deq_now, to_read;
345 stream_session_t *app_session;
348 if (PREDICT_FALSE (SSL_in_init (oc->ssl)))
350 openssl_ctx_handshake_rx (ctx, tls_session);
354 f = tls_session->server_rx_fifo;
355 deq_max = svm_fifo_max_dequeue (f);
356 max_space = max_buf - BIO_ctrl_pending (oc->wbio);
357 max_space = max_space < 0 ? 0 : max_space;
358 deq_now = clib_min (deq_max, max_space);
362 to_read = clib_min (svm_fifo_max_read_chunk (f), deq_now);
363 wrote = BIO_write (oc->wbio, svm_fifo_head (f), to_read);
366 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
369 svm_fifo_dequeue_drop (f, wrote);
372 to_read = clib_min (svm_fifo_max_read_chunk (f), deq_now - wrote);
373 rv = BIO_write (oc->wbio, svm_fifo_head (f), to_read);
376 svm_fifo_dequeue_drop (f, rv);
380 if (svm_fifo_max_dequeue (f))
381 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
385 if (BIO_ctrl_pending (oc->wbio) <= 0)
388 app_session = session_get_from_handle (ctx->app_session_handle);
389 f = app_session->server_rx_fifo;
390 enq_max = svm_fifo_max_enqueue (f);
393 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
397 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max);
398 read = SSL_read (oc->ssl, svm_fifo_tail (f), deq_now);
401 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
404 svm_fifo_enqueue_nocopy (f, read);
405 if (read < enq_max && BIO_ctrl_pending (oc->wbio) > 0)
407 deq_now = clib_min (svm_fifo_max_write_chunk (f), enq_max - read);
408 read = SSL_read (oc->ssl, svm_fifo_tail (f), deq_now);
410 svm_fifo_enqueue_nocopy (f, read);
413 tls_notify_app_enqueue (ctx, app_session);
414 if (BIO_ctrl_pending (oc->wbio) > 0)
415 tls_add_vpp_q_evt (tls_session->server_rx_fifo, FIFO_EVENT_BUILTIN_RX);
421 openssl_ctx_init_client (tls_ctx_t * ctx)
423 char *ciphers = "ALL:!ADH:!LOW:!EXP:!MD5:!RC4-SHA:!DES-CBC3-SHA:@STRENGTH";
424 long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION;
425 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
426 openssl_main_t *om = &openssl_main;
427 stream_session_t *tls_session;
428 const SSL_METHOD *method;
431 method = SSLv23_client_method ();
434 TLS_DBG (1, "SSLv23_method returned null");
438 oc->ssl_ctx = SSL_CTX_new (method);
439 if (oc->ssl_ctx == NULL)
441 TLS_DBG (1, "SSL_CTX_new returned null");
445 SSL_CTX_set_ecdh_auto (oc->ssl_ctx, 1);
446 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
447 rv = SSL_CTX_set_cipher_list (oc->ssl_ctx, (const char *) ciphers);
450 TLS_DBG (1, "Couldn't set cipher");
454 SSL_CTX_set_options (oc->ssl_ctx, flags);
455 SSL_CTX_set_cert_store (oc->ssl_ctx, om->cert_store);
457 oc->ssl = SSL_new (oc->ssl_ctx);
460 TLS_DBG (1, "Couldn't initialize ssl struct");
464 oc->rbio = BIO_new (BIO_s_mem ());
465 oc->wbio = BIO_new (BIO_s_mem ());
467 BIO_set_mem_eof_return (oc->rbio, -1);
468 BIO_set_mem_eof_return (oc->wbio, -1);
470 SSL_set_bio (oc->ssl, oc->wbio, oc->rbio);
471 SSL_set_connect_state (oc->ssl);
473 rv = SSL_set_tlsext_host_name (oc->ssl, ctx->srv_hostname);
476 TLS_DBG (1, "Couldn't set hostname");
481 * 2. Do the first steps in the handshake.
483 TLS_DBG (1, "Initiating handshake for [%u]%u", ctx->c_thread_index,
484 oc->openssl_ctx_index);
486 tls_session = session_get_from_handle (ctx->tls_session_handle);
489 rv = SSL_do_handshake (oc->ssl);
490 err = SSL_get_error (oc->ssl, rv);
491 openssl_try_handshake_write (oc, tls_session);
492 if (err != SSL_ERROR_WANT_WRITE)
496 TLS_DBG (2, "tls state for [%u]%u is su", ctx->c_thread_index,
497 oc->openssl_ctx_index, SSL_state_string_long (oc->ssl));
502 openssl_ctx_init_server (tls_ctx_t * ctx)
504 char *ciphers = "ALL:!ADH:!LOW:!EXP:!MD5:!RC4-SHA:!DES-CBC3-SHA:@STRENGTH";
505 long flags = SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 | SSL_OP_NO_COMPRESSION;
506 openssl_ctx_t *oc = (openssl_ctx_t *) ctx;
507 stream_session_t *tls_session;
508 const SSL_METHOD *method;
512 #ifdef HAVE_OPENSSL_ASYNC
513 openssl_main_t *om = &openssl_main;
514 openssl_resume_handler *handler;
517 app = application_get (ctx->parent_app_index);
518 if (!app->tls_cert || !app->tls_key)
520 TLS_DBG (1, "tls cert and/or key not configured %d",
521 ctx->parent_app_index);
525 method = SSLv23_method ();
526 oc->ssl_ctx = SSL_CTX_new (method);
529 clib_warning ("Unable to create SSL context");
533 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE);
534 #ifdef HAVE_OPENSSL_ASYNC
536 SSL_CTX_set_mode (oc->ssl_ctx, SSL_MODE_ASYNC);
538 SSL_CTX_set_options (oc->ssl_ctx, flags);
539 SSL_CTX_set_ecdh_auto (oc->ssl_ctx, 1);
541 rv = SSL_CTX_set_cipher_list (oc->ssl_ctx, (const char *) ciphers);
544 TLS_DBG (1, "Couldn't set cipher");
549 * Set the key and cert
551 cert_bio = BIO_new (BIO_s_mem ());
552 BIO_write (cert_bio, app->tls_cert, vec_len (app->tls_cert));
553 oc->srvcert = PEM_read_bio_X509 (cert_bio, NULL, NULL, NULL);
556 clib_warning ("unable to parse certificate");
559 SSL_CTX_use_certificate (oc->ssl_ctx, oc->srvcert);
563 cert_bio = BIO_new (BIO_s_mem ());
564 BIO_write (cert_bio, app->tls_key, vec_len (app->tls_key));
565 oc->pkey = PEM_read_bio_PrivateKey (cert_bio, NULL, NULL, NULL);
568 clib_warning ("unable to parse pkey");
571 SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
573 SSL_CTX_use_PrivateKey (oc->ssl_ctx, oc->pkey);
576 /* Start a new connection */
578 oc->ssl = SSL_new (oc->ssl_ctx);
581 TLS_DBG (1, "Couldn't initialize ssl struct");
585 oc->rbio = BIO_new (BIO_s_mem ());
586 oc->wbio = BIO_new (BIO_s_mem ());
588 BIO_set_mem_eof_return (oc->rbio, -1);
589 BIO_set_mem_eof_return (oc->wbio, -1);
591 SSL_set_bio (oc->ssl, oc->wbio, oc->rbio);
592 SSL_set_accept_state (oc->ssl);
594 TLS_DBG (1, "Initiating handshake for [%u]%u", ctx->c_thread_index,
595 oc->openssl_ctx_index);
597 tls_session = session_get_from_handle (ctx->tls_session_handle);
600 rv = SSL_do_handshake (oc->ssl);
601 err = SSL_get_error (oc->ssl, rv);
602 openssl_try_handshake_write (oc, tls_session);
603 #ifdef HAVE_OPENSSL_ASYNC
604 if (err == SSL_ERROR_WANT_ASYNC)
606 handler = (openssl_resume_handler *) openssl_ctx_handshake_rx;
607 vpp_ssl_async_process_event (ctx, handler);
611 if (err != SSL_ERROR_WANT_WRITE)
615 TLS_DBG (2, "tls state for [%u]%u is su", ctx->c_thread_index,
616 oc->openssl_ctx_index, SSL_state_string_long (oc->ssl));
621 openssl_handshake_is_over (tls_ctx_t * ctx)
623 openssl_ctx_t *mc = (openssl_ctx_t *) ctx;
626 return SSL_is_init_finished (mc->ssl);
629 const static tls_engine_vft_t openssl_engine = {
630 .ctx_alloc = openssl_ctx_alloc,
631 .ctx_free = openssl_ctx_free,
632 .ctx_get = openssl_ctx_get,
633 .ctx_get_w_thread = openssl_ctx_get_w_thread,
634 .ctx_init_server = openssl_ctx_init_server,
635 .ctx_init_client = openssl_ctx_init_client,
636 .ctx_write = openssl_ctx_write,
637 .ctx_read = openssl_ctx_read,
638 .ctx_handshake_is_over = openssl_handshake_is_over,
642 tls_init_ca_chain (void)
644 openssl_main_t *om = &openssl_main;
645 tls_main_t *tm = vnet_tls_get_main ();
650 if (access (tm->ca_cert_path, F_OK | R_OK) == -1)
652 clib_warning ("Could not initialize TLS CA certificates");
656 if (!(om->cert_store = X509_STORE_new ()))
658 clib_warning ("failed to create cert store");
662 rv = X509_STORE_load_locations (om->cert_store, tm->ca_cert_path, 0);
665 clib_warning ("failed to load ca certificate");
668 if (tm->use_test_cert_in_ca)
670 cert_bio = BIO_new (BIO_s_mem ());
671 BIO_write (cert_bio, test_srv_crt_rsa, test_srv_crt_rsa_len);
672 testcert = PEM_read_bio_X509 (cert_bio, NULL, NULL, NULL);
675 clib_warning ("unable to parse certificate");
678 X509_STORE_add_cert (om->cert_store, testcert);
681 return (rv < 0 ? -1 : 0);
684 static clib_error_t *
685 tls_openssl_init (vlib_main_t * vm)
687 vlib_thread_main_t *vtm = vlib_get_thread_main ();
688 openssl_main_t *om = &openssl_main;
692 num_threads = 1 /* main thread */ + vtm->n_threads;
694 if ((error = vlib_call_init_function (vm, tls_init)))
698 SSL_load_error_strings ();
700 if (tls_init_ca_chain ())
702 clib_warning ("failed to initialize TLS CA chain");
706 vec_validate (om->ctx_pool, num_threads - 1);
708 tls_register_engine (&openssl_engine, TLS_ENGINE_OPENSSL);
715 #ifdef HAVE_OPENSSL_ASYNC
716 static clib_error_t *
717 tls_openssl_set_command_fn (vlib_main_t * vm, unformat_input_t * input,
718 vlib_cli_command_t * cmd)
720 openssl_main_t *om = &openssl_main;
721 char *engine_name = NULL;
722 char *engine_alg = NULL;
723 u8 engine_name_set = 0;
726 /* By present, it is not allowed to configure engine again after running */
729 clib_warning ("engine has started!\n");
730 return clib_error_return
731 (0, "engine has started, and no config is accepted");
734 while (unformat_check_input (input) != UNFORMAT_END_OF_INPUT)
736 if (unformat (input, "engine %s", &engine_name))
740 else if (unformat (input, "async"))
743 openssl_async_node_enable_disable (1);
745 else if (unformat (input, "alg %s", &engine_alg))
747 for (i = 0; i < strnlen (engine_alg, MAX_CRYPTO_LEN); i++)
748 engine_alg[i] = toupper (engine_alg[i]);
751 return clib_error_return (0, "failed: unknown input `%U'",
752 format_unformat_error, input);
755 /* reset parameters if engine is not configured */
756 if (!engine_name_set)
758 clib_warning ("No engine provided! \n");
763 if (!openssl_engine_register (engine_name, engine_alg))
765 return clib_error_return (0, "failed to register %s polling",
774 VLIB_CLI_COMMAND (tls_openssl_set_command, static) =
776 .path = "tls openssl set",
777 .short_help = "tls openssl set [engine <engine name>] [alg [algorithm] [async]",
778 .function = tls_openssl_set_command_fn,
784 VLIB_INIT_FUNCTION (tls_openssl_init);
787 VLIB_PLUGIN_REGISTER () = {
788 .version = VPP_BUILD_VER,
789 .description = "openssl based TLS Engine",
794 * fd.io coding-style-patch-verification: ON
797 * eval: (c-set-style "gnu")