From 2c876f9c587a359ffd7191723cf616b14dcb9cf1 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Mon, 10 May 2021 21:12:27 -0700 Subject: [PATCH] tls: switch to vc service and track half-open sessions Half-open tls sessions are now tracked by the app worker and are cleaned up only when tcp cleans up its half-open session, i.e., independent of when the established tls context is allocated. Type: improvement Signed-off-by: Florin Coras Change-Id: If5d594d7095192dd527daf4ea1358ffeccdfcc7a --- src/vnet/session/application.h | 2 +- src/vnet/session/application_interface.h | 3 ++ src/vnet/session/application_worker.c | 7 ++- src/vnet/session/session.c | 7 +-- src/vnet/session/session.h | 1 + src/vnet/tls/tls.c | 75 +++++++++++++++++++++++++++----- src/vnet/tls/tls.h | 2 +- 7 files changed, 78 insertions(+), 19 deletions(-) diff --git a/src/vnet/session/application.h b/src/vnet/session/application.h index eca3d3cf2c5..5ddf1d21fe3 100644 --- a/src/vnet/session/application.h +++ b/src/vnet/session/application.h @@ -314,7 +314,7 @@ int app_worker_init_connected (app_worker_t * app_wrk, session_t * s); int app_worker_connect_notify (app_worker_t * app_wrk, session_t * s, session_error_t err, u32 opaque); int app_worker_add_half_open (app_worker_t *app_wrk, session_handle_t sh); -int app_worker_del_half_open (app_worker_t *app_wrk, u32 ho_index); +int app_worker_del_half_open (app_worker_t *app_wrk, session_t *s); int app_worker_close_notify (app_worker_t * app_wrk, session_t * s); int app_worker_transport_closed_notify (app_worker_t * app_wrk, session_t * s); diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index 14db42a0009..733f5dc0ef6 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -53,6 +53,9 @@ typedef struct session_cb_vft_ /** Notify app that session or transport are about to be removed */ void (*session_cleanup_callback) (session_t * s, session_cleanup_ntf_t ntf); + /** Notify app that half open state was cleaned up (optional) */ + void (*half_open_cleanup_callback) (session_t *s); + /** Notify app that session was reset */ void (*session_reset_callback) (session_t * s); diff --git a/src/vnet/session/application_worker.c b/src/vnet/session/application_worker.c index b55c55aea47..fad70c06de3 100644 --- a/src/vnet/session/application_worker.c +++ b/src/vnet/session/application_worker.c @@ -393,10 +393,13 @@ app_worker_add_half_open (app_worker_t *app_wrk, session_handle_t sh) } int -app_worker_del_half_open (app_worker_t *app_wrk, u32 ho_index) +app_worker_del_half_open (app_worker_t *app_wrk, session_t *s) { + application_t *app = application_get (app_wrk->app_index); ASSERT (vlib_get_thread_index () == 0); - pool_put_index (app_wrk->half_open_table, ho_index); + pool_put_index (app_wrk->half_open_table, s->ho_index); + if (app->cb_fns.half_open_cleanup_callback) + app->cb_fns.half_open_cleanup_callback (s); return 0; } diff --git a/src/vnet/session/session.c b/src/vnet/session/session.c index 16b6ce412ac..0d2301c5f83 100644 --- a/src/vnet/session/session.c +++ b/src/vnet/session/session.c @@ -316,7 +316,7 @@ session_half_open_delete_notify (transport_connection_t *tc) s = ho_session_get (tc->s_index); app_wrk = app_worker_get (s->app_wrk_index); - app_worker_del_half_open (app_wrk, s->ho_index); + app_worker_del_half_open (app_wrk, s); session_free (s); } @@ -339,7 +339,7 @@ session_alloc_for_connection (transport_connection_t * tc) return s; } -static session_t * +session_t * session_alloc_for_half_open (transport_connection_t *tc) { session_t *s; @@ -1297,7 +1297,8 @@ session_open_vc (session_endpoint_cfg_t *rmt, session_handle_t *rsh) ho->opaque = rmt->opaque; *rsh = session_handle (ho); - session_lookup_add_half_open (tc, tc->c_index); + if (!(tc->flags & TRANSPORT_CONNECTION_F_NO_LOOKUP)) + session_lookup_add_half_open (tc, tc->c_index); return 0; } diff --git a/src/vnet/session/session.h b/src/vnet/session/session.h index e29cdbfc933..d14c68971de 100644 --- a/src/vnet/session/session.h +++ b/src/vnet/session/session.h @@ -759,6 +759,7 @@ void session_node_enable_disable (u8 is_en); clib_error_t *vnet_session_enable_disable (vlib_main_t * vm, u8 is_en); session_t *session_alloc_for_connection (transport_connection_t * tc); +session_t *session_alloc_for_half_open (transport_connection_t *tc); #endif /* __included_session_h__ */ diff --git a/src/vnet/tls/tls.c b/src/vnet/tls/tls.c index e19f8e6c040..09fc8c62325 100644 --- a/src/vnet/tls/tls.c +++ b/src/vnet/tls/tls.c @@ -123,19 +123,20 @@ tls_ctx_half_open_alloc (void) if (PREDICT_FALSE (will_expand && vlib_num_workers ())) { clib_rwlock_writer_lock (&tm->half_open_rwlock); - pool_get (tm->half_open_ctx_pool, ctx); - ctx_index = ctx - tm->half_open_ctx_pool; + pool_get_zero (tm->half_open_ctx_pool, ctx); + ctx->c_c_index = ctx - tm->half_open_ctx_pool; + ctx_index = ctx->c_c_index; clib_rwlock_writer_unlock (&tm->half_open_rwlock); } else { /* reader lock assumption: only main thread will call pool_get */ clib_rwlock_reader_lock (&tm->half_open_rwlock); - pool_get (tm->half_open_ctx_pool, ctx); - ctx_index = ctx - tm->half_open_ctx_pool; + pool_get_zero (tm->half_open_ctx_pool, ctx); + ctx->c_c_index = ctx - tm->half_open_ctx_pool; + ctx_index = ctx->c_c_index; clib_rwlock_reader_unlock (&tm->half_open_rwlock); } - clib_memset (ctx, 0, sizeof (*ctx)); return ctx_index; } @@ -420,6 +421,20 @@ tls_session_reset_callback (session_t * s) } } +static void +tls_session_cleanup_ho (session_t *s) +{ + tls_ctx_t *ctx; + u32 ho_index; + + /* session opaque stores the opaque passed on connect */ + ho_index = s->opaque; + ctx = tls_ctx_half_open_get (ho_index); + session_half_open_delete_notify (&ctx->connection); + tls_ctx_half_open_reader_unlock (); + tls_ctx_half_open_free (ho_index); +} + int tls_add_segment_callback (u32 client_index, u64 segment_handle) { @@ -533,15 +548,14 @@ tls_session_connected_cb (u32 tls_app_index, u32 ho_ctx_index, app_worker_connect_notify (app_wrk, 0, err, api_context); } tls_ctx_half_open_reader_unlock (); - tls_ctx_half_open_free (ho_ctx_index); return rv; } ctx_handle = tls_ctx_alloc (ho_ctx->tls_ctx_engine); ctx = tls_ctx_get (ctx_handle); clib_memcpy_fast (ctx, ho_ctx, sizeof (*ctx)); + /* Half-open freed on tcp half-open cleanup notification */ tls_ctx_half_open_reader_unlock (); - tls_ctx_half_open_free (ho_ctx_index); ctx->c_thread_index = vlib_get_thread_index (); ctx->tls_ctx_handle = ctx_handle; @@ -660,6 +674,7 @@ static session_cb_vft_t tls_app_cb_vft = { .session_disconnect_callback = tls_session_disconnect_callback, .session_connected_callback = tls_session_connected_callback, .session_reset_callback = tls_session_reset_callback, + .half_open_cleanup_callback = tls_session_cleanup_ho, .add_segment_callback = tls_add_segment_callback, .del_segment_callback = tls_del_segment_callback, .builtin_app_rx_callback = tls_app_rx_callback, @@ -704,6 +719,8 @@ tls_connect (transport_endpoint_cfg_t * tep) ctx->tcp_is_ip4 = sep->is_ip4; ctx->tls_type = sep->transport_proto; ctx->ckpair_index = ccfg->ckpair_index; + ctx->c_proto = TRANSPORT_PROTO_TLS; + ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; if (ccfg->hostname[0]) { ctx->srv_hostname = format (0, "%s", ccfg->hostname); @@ -721,8 +738,11 @@ tls_connect (transport_endpoint_cfg_t * tep) if ((rv = vnet_connect (cargs))) return rv; + /* Track half-open tcp session in case we need to clean it up */ + ctx->tls_session_handle = cargs->sh; + TLS_DBG (1, "New connect request %u engine %d", ctx_index, engine_type); - return 0; + return ctx_index; } void @@ -871,6 +891,26 @@ tls_listener_get (u32 listener_index) return &ctx->connection; } +static transport_connection_t * +tls_half_open_get (u32 ho_index) +{ + tls_main_t *tm = &tls_main; + tls_ctx_t *ctx; + ctx = tls_ctx_half_open_get (ho_index); + clib_rwlock_reader_unlock (&tm->half_open_rwlock); + return &ctx->connection; +} + +static void +tls_cleanup_ho (u32 ho_index) +{ + tls_main_t *tm = &tls_main; + tls_ctx_t *ctx; + ctx = tls_ctx_half_open_get (ho_index); + clib_rwlock_reader_unlock (&tm->half_open_rwlock); + session_cleanup_half_open (ctx->tls_session_handle); +} + int tls_custom_tx_callback (void *session, transport_send_params_t * sp) { @@ -992,10 +1032,19 @@ format_tls_listener (u8 * s, va_list * args) u8 * format_tls_half_open (u8 * s, va_list * args) { - u32 tc_index = va_arg (*args, u32); + u32 ho_index = va_arg (*args, u32); u32 __clib_unused thread_index = va_arg (*args, u32); - tls_ctx_t *ctx = tls_ctx_half_open_get (tc_index); - s = format (s, "[TLS] half-open app %u", ctx->parent_app_wrk_index); + session_t *tcp_ho; + tls_ctx_t *ho_ctx; + + ho_ctx = tls_ctx_half_open_get (ho_index); + + tcp_ho = session_get_from_handle (ho_ctx->tls_session_handle); + s = format (s, "[%d:%d][%s] half-open app_wrk %u engine %u ts %d:%d", + ho_ctx->c_thread_index, ho_ctx->c_s_index, "TLS", + ho_ctx->parent_app_wrk_index, ho_ctx->tls_ctx_engine, + tcp_ho->thread_index, tcp_ho->session_index); + tls_ctx_half_open_reader_unlock (); return s; } @@ -1080,6 +1129,8 @@ static const transport_proto_vft_t tls_proto = { .stop_listen = tls_stop_listen, .get_connection = tls_connection_get, .get_listener = tls_listener_get, + .get_half_open = tls_half_open_get, + .cleanup_ho = tls_cleanup_ho, .custom_tx = tls_custom_tx_callback, .format_connection = format_tls_connection, .format_half_open = format_tls_half_open, @@ -1090,7 +1141,7 @@ static const transport_proto_vft_t tls_proto = { .name = "tls", .short_name = "J", .tx_type = TRANSPORT_TX_INTERNAL, - .service_type = TRANSPORT_SERVICE_APP, + .service_type = TRANSPORT_SERVICE_VC, }, }; diff --git a/src/vnet/tls/tls.h b/src/vnet/tls/tls.h index 2038fdff133..eba70c0a8bb 100644 --- a/src/vnet/tls/tls.h +++ b/src/vnet/tls/tls.h @@ -44,12 +44,12 @@ typedef struct tls_cxt_id_ u32 parent_app_api_ctx; }; session_handle_t tls_session_handle; + void *migrate_ctx; u32 parent_app_wrk_index; u32 ssl_ctx; u32 listener_ctx_index; u8 tcp_is_ip4; u8 tls_engine_id; - void *migrate_ctx; } tls_ctx_id_t; /* *INDENT-ON* */ -- 2.16.6