X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fquic%2Fquic.c;h=31cfcced86fad0c8dbe7050789e06ca6c827a0c4;hb=f22f4e562e1b922cff036ef628b77fd2d479d015;hp=24c5bb4493484cb9526f297207483f74eaeb5b91;hpb=deaf97f4538ac84b4922ef9a14f29699ff898f91;p=vpp.git diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c index 24c5bb44934..31cfcced86f 100644 --- a/src/plugins/quic/quic.c +++ b/src/plugins/quic/quic.c @@ -26,9 +26,14 @@ #include #include #include -#include +#include #include +#include + +#include + +extern quicly_crypto_engine_t quic_crypto_engine; static char *quic_error_strings[] = { #define quic_error(n,s) s, @@ -36,7 +41,9 @@ static char *quic_error_strings[] = { #undef quic_error }; -static quic_main_t quic_main; +#define DEFAULT_MAX_PACKETS_PER_KEY 16777216 + +quic_main_t quic_main; static void quic_update_timer (quic_ctx_t * ctx); static void quic_check_quic_session_connected (quic_ctx_t * ctx); static int quic_reset_connection (u64 udp_session_handle, @@ -47,38 +54,199 @@ static quicly_stream_open_t on_stream_open; static quicly_closed_by_peer_t on_closed_by_peer; static quicly_now_t quicly_vpp_now_cb; +/* Crypto contexts */ + +static inline void +quic_crypto_context_make_key_from_ctx (clib_bihash_kv_24_8_t * kv, + quic_ctx_t * ctx) +{ + application_t *app = application_get (ctx->parent_app_id); + kv->key[0] = ((u64) ctx->ckpair_index) << 32 | (u64) ctx->crypto_engine; + kv->key[1] = app->sm_properties.rx_fifo_size - 1; + kv->key[2] = app->sm_properties.tx_fifo_size - 1; +} + +static inline void +quic_crypto_context_make_key_from_crctx (clib_bihash_kv_24_8_t * kv, + crypto_context_t * crctx) +{ + quic_crypto_context_data_t *data = + (quic_crypto_context_data_t *) crctx->data; + kv->key[0] = ((u64) crctx->ckpair_index) << 32 | (u64) crctx->crypto_engine; + kv->key[1] = data->quicly_ctx.transport_params.max_stream_data.bidi_local; + kv->key[2] = data->quicly_ctx.transport_params.max_stream_data.bidi_remote; +} + +static void +quic_crypto_context_free_if_needed (crypto_context_t * crctx, u8 thread_index) +{ + quic_main_t *qm = &quic_main; + clib_bihash_kv_24_8_t kv; + if (crctx->n_subscribers) + return; + quic_crypto_context_make_key_from_crctx (&kv, crctx); + clib_bihash_add_del_24_8 (&qm->wrk_ctx[thread_index].crypto_context_hash, + &kv, 0 /* is_add */ ); + clib_mem_free (crctx->data); + pool_put (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx); +} + +static quicly_datagram_t * +quic_alloc_packet (quicly_packet_allocator_t * self, size_t payloadsize) +{ + quicly_datagram_t *packet; + if ((packet = + clib_mem_alloc (sizeof (*packet) + payloadsize + + sizeof (quic_encrypt_cb_ctx))) == NULL) + return NULL; + packet->data.base = + (uint8_t *) packet + sizeof (*packet) + sizeof (quic_encrypt_cb_ctx); + quic_encrypt_cb_ctx *encrypt_cb_ctx = + (quic_encrypt_cb_ctx *) ((uint8_t *) packet + sizeof (*packet)); + + clib_memset (encrypt_cb_ctx, 0, sizeof (*encrypt_cb_ctx)); + return packet; +} + +static void +quic_free_packet (quicly_packet_allocator_t * self, + quicly_datagram_t * packet) +{ + clib_mem_free (packet); +} + +quicly_packet_allocator_t quic_packet_allocator = + { quic_alloc_packet, quic_free_packet }; + static int -quic_store_quicly_ctx (application_t * app, u32 ckpair_index, - u8 crypto_engine) +quic_app_cert_key_pair_delete_callback (app_cert_key_pair_t * ckpair) { quic_main_t *qm = &quic_main; - quicly_context_t *quicly_ctx; - ptls_iovec_t key_vec; - app_cert_key_pair_t *ckpair; - u64 max_enq; - if (app->quicly_ctx) - return 0; + crypto_context_t *crctx; + clib_bihash_kv_24_8_t kv; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + int num_threads = 1 /* main thread */ + vtm->n_threads; + int i; - if (crypto_engine == CRYPTO_ENGINE_NONE) + for (i = 0; i < num_threads; i++) { - QUIC_DBG (2, "No crypto engine specified, using %d", crypto_engine); - crypto_engine = qm->default_crypto_engine; + /* *INDENT-OFF* */ + pool_foreach (crctx, qm->wrk_ctx[i].crypto_ctx_pool, ({ + if (crctx->ckpair_index == ckpair->cert_key_index) + { + quic_crypto_context_make_key_from_crctx (&kv, crctx); + clib_bihash_add_del_24_8 (&qm->wrk_ctx[i].crypto_context_hash, &kv, 0 /* is_add */ ); + } + })); + /* *INDENT-ON* */ } - if (!clib_bitmap_get (qm->available_crypto_engines, crypto_engine)) + return 0; +} + +static crypto_context_t * +quic_crypto_context_alloc (u8 thread_index) +{ + quic_main_t *qm = &quic_main; + crypto_context_t *crctx; + u32 idx; + + pool_get (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx); + clib_memset (crctx, 0, sizeof (*crctx)); + idx = (crctx - qm->wrk_ctx[thread_index].crypto_ctx_pool); + crctx->ctx_index = ((u32) thread_index) << 24 | idx; + + return crctx; +} + +static crypto_context_t * +quic_crypto_context_get (u32 cr_index, u32 thread_index) +{ + quic_main_t *qm = &quic_main; + ASSERT (cr_index >> 24 == thread_index); + return pool_elt_at_index (qm->wrk_ctx[thread_index].crypto_ctx_pool, + cr_index & 0x00ffffff); +} + +static clib_error_t * +quic_list_crypto_context_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + quic_main_t *qm = &quic_main; + crypto_context_t *crctx; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + int i, num_threads = 1 /* main thread */ + vtm->n_threads; + for (i = 0; i < num_threads; i++) { - QUIC_DBG (1, "Quic does not support crypto engine %d", crypto_engine); - return VNET_API_ERROR_MISSING_CERT_KEY; + /* *INDENT-OFF* */ + pool_foreach (crctx, qm->wrk_ctx[i].crypto_ctx_pool, ({ + vlib_cli_output (vm, "[%d][Q]%U", i, format_crypto_context, crctx); + })); + /* *INDENT-ON* */ } + return 0; +} + +static clib_error_t * +quic_set_max_packets_per_key_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + quic_main_t *qm = &quic_main; + unformat_input_t _line_input, *line_input = &_line_input; + u64 tmp; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "%U", unformat_memory_size, &tmp)) + { + qm->max_packets_per_key = tmp; + } + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + } + + return 0; +} + +static void +quic_release_crypto_context (u32 crypto_context_index, u8 thread_index) +{ + crypto_context_t *crctx; + crctx = quic_crypto_context_get (crypto_context_index, thread_index); + crctx->n_subscribers--; + quic_crypto_context_free_if_needed (crctx, thread_index); +} + +static int +quic_init_crypto_context (crypto_context_t * crctx, quic_ctx_t * ctx) +{ + quic_main_t *qm = &quic_main; + quicly_context_t *quicly_ctx; + ptls_iovec_t key_vec; + app_cert_key_pair_t *ckpair; + application_t *app; + quic_crypto_context_data_t *data; + ptls_context_t *ptls_ctx; + + QUIC_DBG (2, "Init quic crctx %d thread %d", crctx->ctx_index, + ctx->c_thread_index); + + data = clib_mem_alloc (sizeof (*data)); + /* picotls depends on data being zeroed */ + clib_memset (data, 0, sizeof (*data)); + crctx->data = (void *) data; + quicly_ctx = &data->quicly_ctx; + ptls_ctx = &data->ptls_ctx; - quicly_ctx_data_t *quicly_ctx_data = - clib_mem_alloc (sizeof (quicly_ctx_data_t)); - clib_memset (quicly_ctx_data, 0, sizeof (*quicly_ctx_data)); /* picotls depends on this */ - quicly_ctx = &quicly_ctx_data->quicly_ctx; - ptls_context_t *ptls_ctx = &quicly_ctx_data->ptls_ctx; ptls_ctx->random_bytes = ptls_openssl_random_bytes; ptls_ctx->get_time = &ptls_get_time; ptls_ctx->key_exchanges = ptls_openssl_key_exchanges; - ptls_ctx->cipher_suites = qm->quic_ciphers[crypto_engine]; + ptls_ctx->cipher_suites = qm->quic_ciphers[ctx->crypto_engine]; ptls_ctx->certificates.list = NULL; ptls_ctx->certificates.count = 0; ptls_ctx->esni = NULL; @@ -91,58 +259,113 @@ quic_store_quicly_ctx (application_t * app, u32 ckpair_index, ptls_ctx->hkdf_label_prefix__obsolete = NULL; ptls_ctx->require_dhe_on_psk = 1; ptls_ctx->encrypt_ticket = &qm->session_cache.super; - - app->quicly_ctx = (u64 *) quicly_ctx; clib_memcpy (quicly_ctx, &quicly_spec_context, sizeof (quicly_context_t)); quicly_ctx->max_packet_size = QUIC_MAX_PACKET_SIZE; + quicly_ctx->max_packets_per_key = qm->max_packets_per_key; quicly_ctx->tls = ptls_ctx; quicly_ctx->stream_open = &on_stream_open; quicly_ctx->closed_by_peer = &on_closed_by_peer; quicly_ctx->now = &quicly_vpp_now_cb; quicly_amend_ptls_context (quicly_ctx->tls); + quicly_ctx->packet_allocator = &quic_packet_allocator; + quicly_ctx->crypto_engine = &quic_crypto_engine; quicly_ctx->transport_params.max_data = QUIC_INT_MAX; quicly_ctx->transport_params.max_streams_uni = (uint64_t) 1 << 60; quicly_ctx->transport_params.max_streams_bidi = (uint64_t) 1 << 60; - quicly_ctx->transport_params.idle_timeout = qm->connection_timeout; + quicly_ctx->transport_params.max_idle_timeout = qm->connection_timeout; - /* max_enq is FIFO_SIZE - 1 */ - max_enq = app->sm_properties.rx_fifo_size - 1; - quicly_ctx->transport_params.max_stream_data.bidi_local = max_enq; - max_enq = app->sm_properties.tx_fifo_size - 1; - quicly_ctx->transport_params.max_stream_data.bidi_remote = max_enq; + app = application_get (ctx->parent_app_id); + quicly_ctx->transport_params.max_stream_data.bidi_local = + app->sm_properties.rx_fifo_size - 1; + quicly_ctx->transport_params.max_stream_data.bidi_remote = + app->sm_properties.tx_fifo_size - 1; quicly_ctx->transport_params.max_stream_data.uni = QUIC_INT_MAX; - quicly_ctx->tls->random_bytes (quicly_ctx_data->cid_key, 16); - quicly_ctx_data->cid_key[16] = 0; - key_vec = ptls_iovec_init (quicly_ctx_data->cid_key, - strlen (quicly_ctx_data->cid_key)); + if (!app->quic_iv_set) + { + ptls_openssl_random_bytes (app->quic_iv, QUIC_IV_LEN - 1); + app->quic_iv[QUIC_IV_LEN - 1] = 0; + app->quic_iv_set = 1; + } + + clib_memcpy (data->cid_key, app->quic_iv, QUIC_IV_LEN); + key_vec = ptls_iovec_init (data->cid_key, QUIC_IV_LEN); quicly_ctx->cid_encryptor = quicly_new_default_cid_encryptor (&ptls_openssl_bfecb, &ptls_openssl_aes128ecb, &ptls_openssl_sha256, key_vec); - ckpair = app_cert_key_pair_get_if_valid (ckpair_index); + ckpair = app_cert_key_pair_get_if_valid (crctx->ckpair_index); if (!ckpair || !ckpair->key || !ckpair->cert) { - QUIC_DBG (1, "Wrong ckpair id %d\n", ckpair_index); - goto error; + QUIC_DBG (1, "Wrong ckpair id %d\n", crctx->ckpair_index); + return -1; } if (load_bio_private_key (quicly_ctx->tls, (char *) ckpair->key)) { QUIC_DBG (1, "failed to read private key from app configuration\n"); - goto error; + return -1; } if (load_bio_certificate_chain (quicly_ctx->tls, (char *) ckpair->cert)) { QUIC_DBG (1, "failed to load certificate\n"); - goto error; + return -1; } return 0; +} + +static int +quic_acquire_crypto_context (quic_ctx_t * ctx) +{ + quic_main_t *qm = &quic_main; + crypto_context_t *crctx; + clib_bihash_kv_24_8_t kv; + + if (ctx->crypto_engine == CRYPTO_ENGINE_NONE) + { + QUIC_DBG (2, "No crypto engine specified, using %d", + qm->default_crypto_engine); + ctx->crypto_engine = qm->default_crypto_engine; + } + if (!clib_bitmap_get (qm->available_crypto_engines, ctx->crypto_engine)) + { + QUIC_DBG (1, "Quic does not support crypto engine %d", + ctx->crypto_engine); + return VNET_API_ERROR_MISSING_CERT_KEY; + } + + /* Check for exisiting crypto ctx */ + quic_crypto_context_make_key_from_ctx (&kv, ctx); + if (clib_bihash_search_24_8 + (&qm->wrk_ctx[ctx->c_thread_index].crypto_context_hash, &kv, &kv) == 0) + { + crctx = quic_crypto_context_get (kv.value, ctx->c_thread_index); + QUIC_DBG (2, "Found exisiting crypto context %d", kv.value); + ctx->crypto_context_index = kv.value; + crctx->n_subscribers++; + return 0; + } + + crctx = quic_crypto_context_alloc (ctx->c_thread_index); + ctx->crypto_context_index = crctx->ctx_index; + kv.value = crctx->ctx_index; + crctx->crypto_engine = ctx->crypto_engine; + crctx->ckpair_index = ctx->ckpair_index; + if (quic_init_crypto_context (crctx, ctx)) + goto error; + if (vnet_app_add_cert_key_interest (ctx->ckpair_index, qm->app_index)) + goto error; + crctx->n_subscribers++; + clib_bihash_add_del_24_8 (&qm-> + wrk_ctx[ctx->c_thread_index].crypto_context_hash, + &kv, 1 /* is_add */ ); + return 0; + error: - clib_mem_free (quicly_ctx_data); + quic_crypto_context_free_if_needed (crctx, ctx->c_thread_index); return VNET_API_ERROR_MISSING_CERT_KEY; } @@ -189,7 +412,7 @@ quic_ctx_get_if_valid (u32 ctx_index, u32 thread_index) return pool_elt_at_index (quic_main.ctx_pool[thread_index], ctx_index); } -static quic_ctx_t * +quic_ctx_t * quic_get_conn_ctx (quicly_conn_t * conn) { u64 conn_data; @@ -253,7 +476,11 @@ quic_sendable_packet_count (session_t * udp_session) static quicly_context_t * quic_get_quicly_ctx_from_ctx (quic_ctx_t * ctx) { - return ctx->quicly_ctx; + crypto_context_t *crctx = + quic_crypto_context_get (ctx->crypto_context_index, ctx->c_thread_index); + quic_crypto_context_data_t *data = + (quic_crypto_context_data_t *) crctx->data; + return &data->quicly_ctx; } static quicly_context_t * @@ -262,7 +489,7 @@ quic_get_quicly_ctx_from_udp (u64 udp_session_handle) session_t *udp_session = session_get_from_handle (udp_session_handle); quic_ctx_t *ctx = quic_ctx_get (udp_session->opaque, udp_session->thread_index); - return ctx->quicly_ctx; + return quic_get_quicly_ctx_from_ctx (ctx); } static inline void @@ -468,13 +695,14 @@ quic_send_datagram (session_t * udp_session, quicly_datagram_t * packet) static int quic_send_packets (quic_ctx_t * ctx) { + quic_main_t *qm = &quic_main; quicly_datagram_t *packets[QUIC_SEND_PACKET_VEC_SIZE]; session_t *udp_session; quicly_conn_t *conn; size_t num_packets, i, max_packets; quicly_packet_allocator_t *pa; - quicly_context_t *quicly_context; int err = 0; + u32 thread_index = vlib_get_thread_index (); /* We have sctx, get qctx */ if (quic_ctx_is_stream (ctx)) @@ -495,15 +723,7 @@ quic_send_packets (quic_ctx_t * ctx) if (quic_sendable_packet_count (udp_session) < 2) goto stop_sending; - quicly_context = quic_get_quicly_ctx_from_ctx (ctx); - if (!quicly_context) - { - clib_warning ("Tried to send packets on non existing app worker %u", - ctx->parent_app_wrk_id); - quic_connection_delete (ctx); - return 1; - } - pa = quicly_context->packet_allocator; + pa = quic_get_quicly_ctx_from_ctx (ctx)->packet_allocator; do { max_packets = quic_sendable_packet_count (udp_session); @@ -513,8 +733,12 @@ quic_send_packets (quic_ctx_t * ctx) if ((err = quicly_send (conn, packets, &num_packets))) goto quicly_error; + quic_crypto_batch_tx_packets (&qm->wrk_ctx + [thread_index].crypto_context_batch); + for (i = 0; i != num_packets; ++i) { + quic_crypto_finalize_send_packet (packets[i]); if ((err = quic_send_datagram (udp_session, packets[i]))) goto quicly_error; @@ -692,10 +916,10 @@ int quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst, size_t * len, int *wrote_all) { - u32 deq_max, first_deq, max_rd_chunk, rem_offset; quic_stream_data_t *stream_data; session_t *stream_session; svm_fifo_t *f; + u32 deq_max; stream_data = (quic_stream_data_t *) stream->data; stream_session = get_stream_session_from_stream (stream); @@ -719,22 +943,7 @@ quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst, if (off + *len > stream_data->app_tx_data_len) stream_data->app_tx_data_len = off + *len; - /* TODO, use something like : return svm_fifo_peek (f, off, *len, dst); */ - max_rd_chunk = svm_fifo_max_read_chunk (f); - - first_deq = 0; - if (off < max_rd_chunk) - { - first_deq = clib_min (*len, max_rd_chunk - off); - clib_memcpy_fast (dst, svm_fifo_head (f) + off, first_deq); - } - - if (max_rd_chunk < off + *len) - { - rem_offset = max_rd_chunk < off ? off - max_rd_chunk : 0; - clib_memcpy_fast (dst + first_deq, f->head_chunk->data + rem_offset, - *len - first_deq); - } + svm_fifo_peek (f, off, *len, dst); return 0; } @@ -792,6 +1001,8 @@ quic_on_stream_open (quicly_stream_open_t * self, quicly_stream_t * stream) sctx->stream = stream; sctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; sctx->flags |= QUIC_F_IS_STREAM; + if (quicly_stream_is_unidirectional (stream->stream_id)) + stream_session->flags |= SESSION_F_UNIDIRECTIONAL; stream_data = (quic_stream_data_t *) stream->data; stream_data->ctx_id = sctx_id; @@ -964,9 +1175,8 @@ quic_expired_timers_dispatch (u32 * expired_timers) } /* Transport proto functions */ - static int -quic_connect_stream (session_t * quic_session, u32 opaque) +quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) { uint64_t quic_session_handle; session_t *stream_session; @@ -1019,7 +1229,9 @@ quic_connect_stream (session_t * quic_session, u32 opaque) if (!conn || !quicly_connection_is_ready (conn)) return -1; - if ((rv = quicly_open_stream (conn, &stream, 0 /* uni */ ))) + if ((rv = + quicly_open_stream (conn, &stream, + sep->flags & SESSION_F_UNIDIRECTIONAL))) { QUIC_DBG (2, "Stream open failed with %d", rv); return -1; @@ -1038,6 +1250,8 @@ quic_connect_stream (session_t * quic_session, u32 opaque) stream_session->listener_handle = quic_session_handle; stream_session->session_type = session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, qctx->udp_is_ip4); + if (sep->flags & SESSION_F_UNIDIRECTIONAL) + stream_session->flags |= SESSION_F_UNIDIRECTIONAL; sctx->c_s_index = stream_session->session_index; stream_data = (quic_stream_data_t *) stream->data; @@ -1052,14 +1266,14 @@ quic_connect_stream (session_t * quic_session, u32 opaque) { QUIC_ERR ("failed to app_worker_init_connected"); quicly_reset_stream (stream, QUIC_APP_CONNECT_NOTIFY_ERROR); - return app_worker_connect_notify (app_wrk, NULL, opaque); + return app_worker_connect_notify (app_wrk, NULL, sep->opaque); } svm_fifo_add_want_deq_ntf (stream_session->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL | SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); - if (app_worker_connect_notify (app_wrk, stream_session, opaque)) + if (app_worker_connect_notify (app_wrk, stream_session, sep->opaque)) { QUIC_ERR ("failed to notify app"); quic_increment_counter (QUIC_ERROR_CLOSED_STREAM, 1); @@ -1111,12 +1325,10 @@ quic_connect_connection (session_endpoint_cfg_t * sep) ctx->parent_app_id = app_wrk->app_index; cargs->sep_ext.ns_index = app->ns_index; - if ((error = - quic_store_quicly_ctx (app, sep->ckpair_index, sep->crypto_engine))) + ctx->crypto_engine = sep->crypto_engine; + ctx->ckpair_index = sep->ckpair_index; + if ((error = quic_acquire_crypto_context (ctx))) return error; - /* Also store it in ctx for convenience - * Waiting for crypto_ctx logic */ - ctx->quicly_ctx = (quicly_context_t *) app->quicly_ctx; if ((error = vnet_connect (cargs))) return error; @@ -1134,7 +1346,7 @@ quic_connect (transport_endpoint_cfg_t * tep) quic_session = session_get_from_handle_if_valid (sep->parent_handle); if (quic_session) - return quic_connect_stream (quic_session, sep->opaque); + return quic_connect_stream (quic_session, sep); else return quic_connect_connection (sep); } @@ -1153,6 +1365,9 @@ quic_proto_on_close (u32 ctx_index, u32 thread_index) if (quic_ctx_is_stream (ctx)) { quicly_stream_t *stream = ctx->stream; + if (!quicly_stream_has_send_side (quicly_is_client (stream->conn), + stream->stream_id)) + return; quicly_reset_stream (stream, QUIC_APP_ERROR_CLOSE_NOTIFY); quic_send_packets (ctx); return; @@ -1212,14 +1427,11 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) app = application_get (app_wrk->app_index); QUIC_DBG (2, "Called quic_start_listen for app %d", app_wrk->app_index); - if (quic_store_quicly_ctx (app, sep->ckpair_index, sep->crypto_engine)) - return -1; - - sep->transport_proto = TRANSPORT_PROTO_UDPC; clib_memset (args, 0, sizeof (*args)); args->app_index = qm->app_index; args->sep_ext = *sep; args->sep_ext.ns_index = app->ns_index; + args->sep_ext.transport_proto = TRANSPORT_PROTO_UDPC; if ((rv = vnet_listen (args))) return rv; @@ -1231,9 +1443,6 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) lctx = quic_ctx_get (lctx_index, 0); lctx->flags |= QUIC_F_IS_LISTENER; - /* Also store it in ctx for convenience - * Waiting for crypto_ctx logic */ - lctx->quicly_ctx = (quicly_context_t *) app->quicly_ctx; clib_memcpy (&lctx->c_rmt_ip, &args->sep.peer.ip, sizeof (ip46_address_t)); clib_memcpy (&lctx->c_lcl_ip, &args->sep.ip, sizeof (ip46_address_t)); @@ -1246,6 +1455,10 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) lctx->parent_app_id = app_wrk->app_index; lctx->udp_session_handle = udp_handle; lctx->c_s_index = quic_listen_session_index; + lctx->crypto_engine = sep->crypto_engine; + lctx->ckpair_index = sep->ckpair_index; + if (quic_acquire_crypto_context (lctx)) + return -1; QUIC_DBG (2, "Listening UDP session 0x%lx", session_handle (udp_listen_session)); @@ -1268,8 +1481,8 @@ quic_stop_listen (u32 lctx_index) if (vnet_unlisten (&a)) clib_warning ("unlisten errored"); - /* TODO: crypto state cleanup */ - + quic_release_crypto_context (lctx->crypto_context_index, + 0 /* thread_index */ ); quic_ctx_free (lctx); return 0; } @@ -1446,6 +1659,19 @@ quic_check_quic_session_connected (quic_ctx_t * ctx) quic_on_quic_session_connected (ctx); } +static inline void +quic_update_conn_ctx (quicly_conn_t * conn, quicly_context_t * quicly_context) +{ + /* we need to update the quicly_conn on migrate + * as it contains a pointer to the crypto context */ + ptls_context_t **tls; + quicly_context_t **_quicly_context; + _quicly_context = (quicly_context_t **) conn; + *_quicly_context = quicly_context; + tls = (ptls_context_t **) quicly_get_tls (conn); + *tls = quicly_context->tls; +} + static void quic_receive_connection (void *arg) { @@ -1453,6 +1679,7 @@ quic_receive_connection (void *arg) quic_ctx_t *temp_ctx, *new_ctx; clib_bihash_kv_16_8_t kv; quicly_conn_t *conn; + quicly_context_t *quicly_context; session_t *udp_session; temp_ctx = arg; @@ -1467,8 +1694,12 @@ quic_receive_connection (void *arg) new_ctx->c_thread_index = thread_index; new_ctx->c_c_index = new_ctx_id; + quic_acquire_crypto_context (new_ctx); conn = new_ctx->conn; + quicly_context = quic_get_quicly_ctx_from_ctx (new_ctx); + quic_update_conn_ctx (conn, quicly_context); + quic_store_conn_ctx (conn, new_ctx); quic_make_connection_key (&kv, quicly_get_master_id (conn)); kv.value = ((u64) thread_index) << 32 | (u64) new_ctx_id; @@ -1500,6 +1731,7 @@ quic_transfer_connection (u32 ctx_index, u32 dest_thread) clib_memcpy (temp_ctx, ctx, sizeof (quic_ctx_t)); quic_stop_ctx_timer (ctx); + quic_release_crypto_context (ctx->crypto_context_index, thread_index); quic_ctx_free (ctx); /* Send connection to destination thread */ @@ -1557,10 +1789,10 @@ quic_udp_session_connected_callback (u32 quic_app_index, u32 ctx_index, quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx); ret = quicly_connect (&ctx->conn, quicly_ctx, (char *) ctx->srv_hostname, - sa, NULL, &quic_main.next_cid, ptls_iovec_init (NULL, - 0), - &quic_main.hs_properties, NULL); - ++quic_main.next_cid.master_id; + sa, NULL, &quic_main.wrk_ctx[thread_index].next_cid, + ptls_iovec_init (NULL, 0), &quic_main.hs_properties, + NULL); + ++quic_main.wrk_ctx[thread_index].next_cid.master_id; /* Save context handle in quicly connection */ quic_store_conn_ctx (ctx->conn, ctx); assert (ret == 0); @@ -1598,6 +1830,8 @@ quic_udp_session_cleanup_callback (session_t * udp_session, ctx = quic_ctx_get (udp_session->opaque, udp_session->thread_index); quic_stop_ctx_timer (ctx); + quic_release_crypto_context (ctx->crypto_context_index, + ctx->c_thread_index); quic_ctx_free (ctx); } @@ -1613,7 +1847,7 @@ quic_udp_session_migrate_callback (session_t * s, session_handle_t new_sh) u32 new_thread = session_thread_from_handle (new_sh); quic_ctx_t *ctx; - QUIC_ERR ("Session %x migrated to %lx", s->session_index, new_sh); + QUIC_DBG (2, "Session %x migrated to %lx", s->session_index, new_sh); QUIC_ASSERT (vlib_get_thread_index () == s->thread_index); ctx = quic_ctx_get (s->opaque, s->thread_index); QUIC_ASSERT (ctx->udp_session_handle == session_handle (s)); @@ -1654,10 +1888,9 @@ quic_udp_session_accepted_callback (session_t * udp_session) ctx->conn_state = QUIC_CONN_STATE_OPENED; ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; - /* Also store it in ctx for convenience - * Waiting for crypto_ctx logic */ - ctx->quicly_ctx = lctx->quicly_ctx; - + ctx->crypto_engine = lctx->crypto_engine; + ctx->ckpair_index = lctx->ckpair_index; + quic_acquire_crypto_context (ctx); udp_session->opaque = ctx_index; /* TODO timeout to delete these if they never connect */ @@ -1792,10 +2025,9 @@ quic_find_packet_ctx (quic_rx_packet_ctx_t * pctx, u32 caller_thread_index) return QUIC_PACKET_TYPE_RECEIVE; } -static int -quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) +static void +quic_accept_connection (quic_rx_packet_ctx_t * pctx) { - u32 thread_index = vlib_get_thread_index (); quicly_context_t *quicly_ctx; session_t *quic_session; clib_bihash_kv_16_8_t kv; @@ -1807,26 +2039,28 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) /* new connection, accept and create context if packet is valid * TODO: check if socket is actually listening? */ - ctx = quic_ctx_get (ctx_index, thread_index); + ctx = quic_ctx_get (pctx->ctx_index, pctx->thread_index); if (ctx->c_s_index != QUIC_SESSION_INVALID) { - QUIC_DBG (2, "already accepted ctx 0x%x", ctx_index); - return 0; + QUIC_DBG (2, "already accepted ctx 0x%x", ctx->c_s_index); + return; } quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx); if ((rv = quicly_accept (&conn, quicly_ctx, NULL, &pctx->sa, - &pctx->packet, NULL, &quic_main.next_cid, NULL))) + &pctx->packet, NULL, + &quic_main.wrk_ctx[pctx->thread_index].next_cid, + NULL))) { /* Invalid packet, pass */ assert (conn == NULL); QUIC_ERR ("Accept failed with %U", quic_format_err, rv); /* TODO: cleanup created quic ctx and UDP session */ - return 0; + return; } assert (conn != NULL); - ++quic_main.next_cid.master_id; + ++quic_main.wrk_ctx[pctx->thread_index].next_cid.master_id; /* Save ctx handle in quicly connection */ quic_store_conn_ctx (conn, ctx); ctx->conn = conn; @@ -1847,7 +2081,7 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) /* Register connection in connections map */ quic_make_connection_key (&kv, quicly_get_master_id (conn)); - kv.value = ((u64) thread_index) << 32 | (u64) ctx_index; + kv.value = ((u64) pctx->thread_index) << 32 | (u64) pctx->ctx_index; clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 1 /* is_add */ ); QUIC_DBG (2, "Registering conn with id %lu %lu", kv.key[0], kv.key[1]); @@ -1855,23 +2089,19 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) if ((rv = app_worker_init_accepted (quic_session))) { QUIC_ERR ("failed to allocate fifos"); - quic_proto_on_close (ctx_index, thread_index); - return rv; + quic_proto_on_close (pctx->ctx_index, pctx->thread_index); + return; } app_wrk = app_worker_get (quic_session->app_wrk_index); if ((rv = app_worker_accept_notify (app_wrk, quic_session))) { QUIC_ERR ("failed to notify accept worker app"); - quic_proto_on_close (ctx_index, thread_index); - return rv; + quic_proto_on_close (pctx->ctx_index, pctx->thread_index); + return; } ctx->conn_state = QUIC_CONN_STATE_READY; - pctx->ctx_index = ctx_index; - pctx->thread_index = thread_index; - - return 0; } static int @@ -1910,6 +2140,7 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, u32 thread_index = vlib_get_thread_index (); u32 cur_deq = svm_fifo_max_dequeue (f) - fifo_offset; quicly_context_t *quicly_ctx; + session_t *udp_session; int rv; ret = svm_fifo_peek (f, fifo_offset, @@ -1949,6 +2180,8 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, if (rv == QUIC_PACKET_TYPE_RECEIVE) { pctx->ptype = QUIC_PACKET_TYPE_RECEIVE; + quic_ctx_t *qctx = quic_ctx_get (pctx->ctx_index, thread_index); + quic_crypto_decrypt_packet (qctx, pctx); return 0; } else if (rv == QUIC_PACKET_TYPE_MIGRATE) @@ -1959,6 +2192,9 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, else if (QUICLY_PACKET_IS_LONG_HEADER (pctx->packet.octets.base[0])) { pctx->ptype = QUIC_PACKET_TYPE_ACCEPT; + udp_session = session_get_from_handle (udp_session_handle); + pctx->ctx_index = udp_session->opaque; + pctx->thread_index = thread_index; } else { @@ -1971,7 +2207,8 @@ static int quic_udp_session_rx_callback (session_t * udp_session) { /* Read data from UDP rx_fifo and pass it to the quicly conn. */ - quic_ctx_t *ctx = NULL; + quic_main_t *qm = &quic_main; + quic_ctx_t *ctx = NULL, *prev_ctx = NULL; svm_fifo_t *f = udp_session->rx_fifo; u32 max_deq; u64 udp_session_handle = session_handle (udp_session); @@ -1995,6 +2232,11 @@ rx_start: fifo_offset = 0; max_packets = QUIC_RCV_MAX_BATCH_PACKETS; +#if CLIB_DEBUG > 0 + clib_memset (packets_ctx, 0xfa, + QUIC_RCV_MAX_BATCH_PACKETS * sizeof (quic_rx_packet_ctx_t)); +#endif + for (i = 0; i < max_packets; i++) { packets_ctx[i].thread_index = UINT32_MAX; @@ -2025,6 +2267,9 @@ rx_start: } } + quic_crypto_batch_rx_packets (&qm-> + wrk_ctx[thread_index].crypto_context_batch); + for (i = 0; i < max_packets; i++) { switch (packets_ctx[i].ptype) @@ -2040,20 +2285,17 @@ rx_start: } break; case QUIC_PACKET_TYPE_ACCEPT: - udp_session = session_get_from_handle (udp_session_handle); - if ((rv = quic_accept_connection (udp_session->opaque, - &packets_ctx[i]))) - { - QUIC_ERR ("quic accept errored with %d", rv); - } + quic_accept_connection (&packets_ctx[i]); break; case QUIC_PACKET_TYPE_RESET: quic_reset_connection (udp_session_handle, &packets_ctx[i]); break; } } + ctx = prev_ctx = NULL; for (i = 0; i < max_packets; i++) { + prev_ctx = ctx; switch (packets_ctx[i].ptype) { case QUIC_PACKET_TYPE_RECEIVE: @@ -2068,9 +2310,11 @@ rx_start: packets_ctx[i].thread_index); break; default: - continue; + continue; /* this exits the for loop since other packet types are + necessarily the last in the batch */ } - quic_send_packets (ctx); + if (ctx != prev_ctx) + quic_send_packets (ctx); } udp_session = session_get_from_handle (udp_session_handle); /* session alloc might have happened */ @@ -2132,6 +2376,7 @@ static session_cb_vft_t quic_app_cb_vft = { .del_segment_callback = quic_del_segment_callback, .builtin_app_rx_callback = quic_udp_session_rx_callback, .session_cleanup_callback = quic_udp_session_cleanup_callback, + .app_cert_key_pair_delete_callback = quic_app_cert_key_pair_delete_callback, }; static const transport_proto_vft_t quic_proto = { @@ -2225,19 +2470,24 @@ quic_init (vlib_main_t * vm) vec_validate (qm->ctx_pool, num_threads - 1); vec_validate (qm->wrk_ctx, num_threads - 1); - /* Timer wheels, one per thread. */ + for (i = 0; i < num_threads; i++) { + qm->wrk_ctx[i].next_cid.thread_id = i; tw = &qm->wrk_ctx[i].timer_wheel; tw_timer_wheel_init_1t_3w_1024sl_ov (tw, quic_expired_timers_dispatch, 1e-3 /* timer period 1ms */ , ~0); tw->last_run_time = vlib_time_now (vlib_get_main ()); + clib_bihash_init_24_8 (&qm->wrk_ctx[i].crypto_context_hash, + "quic crypto contexts", 64, 128 << 10); + + qm->wrk_ctx[i].crypto_context_batch.nb_rx_packets = 0; + qm->wrk_ctx[i].crypto_context_batch.nb_tx_packets = 0; } clib_bihash_init_16_8 (&qm->connection_hash, "quic connections", 1024, 4 << 20); - qm->app_index = a->app_index; qm->tstamp_ticks_per_clock = vm->clib_time.seconds_per_clock / QUIC_TSTAMP_RESOLUTION; @@ -2253,7 +2503,9 @@ quic_init (vlib_main_t * vm) quic_register_cipher_suite (CRYPTO_ENGINE_VPP, quic_crypto_cipher_suites); quic_register_cipher_suite (CRYPTO_ENGINE_PICOTLS, ptls_openssl_cipher_suites); - qm->default_crypto_engine = CRYPTO_ENGINE_PICOTLS; + qm->default_crypto_engine = CRYPTO_ENGINE_VPP; + qm->max_packets_per_key = DEFAULT_MAX_PACKETS_PER_KEY; + clib_rwlock_init (&qm->crypto_keys_quic_rw_lock); vec_free (a->name); return 0; } @@ -2566,6 +2818,18 @@ VLIB_CLI_COMMAND(quic_show_ctx_command, static)= .short_help = "show quic", .function = quic_show_connections_command_fn, }; +VLIB_CLI_COMMAND (quic_list_crypto_context_command, static) = +{ + .path = "show quic crypto context", + .short_help = "list quic crypto contextes", + .function = quic_list_crypto_context_command_fn, +}; +VLIB_CLI_COMMAND (quic_set_max_packets_per_key, static) = +{ + .path = "set quic max_packets_per_key", + .short_help = "set quic max_packets_per_key 16777216", + .function = quic_set_max_packets_per_key_fn, +}; VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER,