X-Git-Url: https://gerrit.fd.io/r/gitweb?p=vpp.git;a=blobdiff_plain;f=src%2Fplugins%2Fquic%2Fquic.c;h=03e46c60616e178fcef3f3d10b2f8dde01c5d4c0;hp=d1f188443d69fa9732d0b3c14b853e3a2e30145c;hb=07063b8ea;hpb=d9577b4aa3a325e46c09796835d22122bec9e3d0 diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c index d1f188443d6..03e46c60616 100644 --- a/src/plugins/quic/quic.c +++ b/src/plugins/quic/quic.c @@ -26,9 +26,14 @@ #include #include #include -#include +#include #include +#include + +#include + +extern quicly_crypto_engine_t quic_crypto_engine; static char *quic_error_strings[] = { #define quic_error(n,s) s, @@ -36,7 +41,9 @@ static char *quic_error_strings[] = { #undef quic_error }; -static quic_main_t quic_main; +#define DEFAULT_MAX_PACKETS_PER_KEY 16777216 + +quic_main_t quic_main; static void quic_update_timer (quic_ctx_t * ctx); static void quic_check_quic_session_connected (quic_ctx_t * ctx); static int quic_reset_connection (u64 udp_session_handle, @@ -47,38 +54,199 @@ static quicly_stream_open_t on_stream_open; static quicly_closed_by_peer_t on_closed_by_peer; static quicly_now_t quicly_vpp_now_cb; +/* Crypto contexts */ + +static inline void +quic_crypto_context_make_key_from_ctx (clib_bihash_kv_24_8_t * kv, + quic_ctx_t * ctx) +{ + application_t *app = application_get (ctx->parent_app_id); + kv->key[0] = ((u64) ctx->ckpair_index) << 32 | (u64) ctx->crypto_engine; + kv->key[1] = app->sm_properties.rx_fifo_size - 1; + kv->key[2] = app->sm_properties.tx_fifo_size - 1; +} + +static inline void +quic_crypto_context_make_key_from_crctx (clib_bihash_kv_24_8_t * kv, + crypto_context_t * crctx) +{ + quic_crypto_context_data_t *data = + (quic_crypto_context_data_t *) crctx->data; + kv->key[0] = ((u64) crctx->ckpair_index) << 32 | (u64) crctx->crypto_engine; + kv->key[1] = data->quicly_ctx.transport_params.max_stream_data.bidi_local; + kv->key[2] = data->quicly_ctx.transport_params.max_stream_data.bidi_remote; +} + +static void +quic_crypto_context_free_if_needed (crypto_context_t * crctx, u8 thread_index) +{ + quic_main_t *qm = &quic_main; + clib_bihash_kv_24_8_t kv; + if (crctx->n_subscribers) + return; + quic_crypto_context_make_key_from_crctx (&kv, crctx); + clib_bihash_add_del_24_8 (&qm->wrk_ctx[thread_index].crypto_context_hash, + &kv, 0 /* is_add */ ); + clib_mem_free (crctx->data); + pool_put (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx); +} + +static quicly_datagram_t * +quic_alloc_packet (quicly_packet_allocator_t * self, size_t payloadsize) +{ + quicly_datagram_t *packet; + if ((packet = + clib_mem_alloc (sizeof (*packet) + payloadsize + + sizeof (quic_encrypt_cb_ctx))) == NULL) + return NULL; + packet->data.base = + (uint8_t *) packet + sizeof (*packet) + sizeof (quic_encrypt_cb_ctx); + quic_encrypt_cb_ctx *encrypt_cb_ctx = + (quic_encrypt_cb_ctx *) ((uint8_t *) packet + sizeof (*packet)); + + clib_memset (encrypt_cb_ctx, 0, sizeof (*encrypt_cb_ctx)); + return packet; +} + +static void +quic_free_packet (quicly_packet_allocator_t * self, + quicly_datagram_t * packet) +{ + clib_mem_free (packet); +} + +quicly_packet_allocator_t quic_packet_allocator = + { quic_alloc_packet, quic_free_packet }; + static int -quic_store_quicly_ctx (application_t * app, u32 ckpair_index, - u8 crypto_engine) +quic_app_cert_key_pair_delete_callback (app_cert_key_pair_t * ckpair) { quic_main_t *qm = &quic_main; - quicly_context_t *quicly_ctx; - ptls_iovec_t key_vec; - app_cert_key_pair_t *ckpair; - u64 max_enq; - if (app->quicly_ctx) - return 0; + crypto_context_t *crctx; + clib_bihash_kv_24_8_t kv; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + int num_threads = 1 /* main thread */ + vtm->n_threads; + int i; - if (crypto_engine == CRYPTO_ENGINE_NONE) + for (i = 0; i < num_threads; i++) { - QUIC_DBG (2, "No crypto engine specified, using %d", crypto_engine); - crypto_engine = qm->default_crypto_engine; + /* *INDENT-OFF* */ + pool_foreach (crctx, qm->wrk_ctx[i].crypto_ctx_pool, ({ + if (crctx->ckpair_index == ckpair->cert_key_index) + { + quic_crypto_context_make_key_from_crctx (&kv, crctx); + clib_bihash_add_del_24_8 (&qm->wrk_ctx[i].crypto_context_hash, &kv, 0 /* is_add */ ); + } + })); + /* *INDENT-ON* */ } - if (!clib_bitmap_get (qm->available_crypto_engines, crypto_engine)) + return 0; +} + +static crypto_context_t * +quic_crypto_context_alloc (u8 thread_index) +{ + quic_main_t *qm = &quic_main; + crypto_context_t *crctx; + u32 idx; + + pool_get (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx); + clib_memset (crctx, 0, sizeof (*crctx)); + idx = (crctx - qm->wrk_ctx[thread_index].crypto_ctx_pool); + crctx->ctx_index = ((u32) thread_index) << 24 | idx; + + return crctx; +} + +static crypto_context_t * +quic_crypto_context_get (u32 cr_index, u32 thread_index) +{ + quic_main_t *qm = &quic_main; + ASSERT (cr_index >> 24 == thread_index); + return pool_elt_at_index (qm->wrk_ctx[thread_index].crypto_ctx_pool, + cr_index & 0x00ffffff); +} + +static clib_error_t * +quic_list_crypto_context_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + quic_main_t *qm = &quic_main; + crypto_context_t *crctx; + vlib_thread_main_t *vtm = vlib_get_thread_main (); + int i, num_threads = 1 /* main thread */ + vtm->n_threads; + for (i = 0; i < num_threads; i++) { - QUIC_DBG (1, "Quic does not support crypto engine %d", crypto_engine); - return VNET_API_ERROR_MISSING_CERT_KEY; + /* *INDENT-OFF* */ + pool_foreach (crctx, qm->wrk_ctx[i].crypto_ctx_pool, ({ + vlib_cli_output (vm, "[%d][Q]%U", i, format_crypto_context, crctx); + })); + /* *INDENT-ON* */ } + return 0; +} + +static clib_error_t * +quic_set_max_packets_per_key_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + quic_main_t *qm = &quic_main; + unformat_input_t _line_input, *line_input = &_line_input; + u64 tmp; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "%U", unformat_memory_size, &tmp)) + { + qm->max_packets_per_key = tmp; + } + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + } + + return 0; +} + +static void +quic_release_crypto_context (u32 crypto_context_index, u8 thread_index) +{ + crypto_context_t *crctx; + crctx = quic_crypto_context_get (crypto_context_index, thread_index); + crctx->n_subscribers--; + quic_crypto_context_free_if_needed (crctx, thread_index); +} + +static int +quic_init_crypto_context (crypto_context_t * crctx, quic_ctx_t * ctx) +{ + quic_main_t *qm = &quic_main; + quicly_context_t *quicly_ctx; + ptls_iovec_t key_vec; + app_cert_key_pair_t *ckpair; + application_t *app; + quic_crypto_context_data_t *data; + ptls_context_t *ptls_ctx; + + QUIC_DBG (2, "Init quic crctx %d thread %d", crctx->ctx_index, + ctx->c_thread_index); + + data = clib_mem_alloc (sizeof (*data)); + /* picotls depends on data being zeroed */ + clib_memset (data, 0, sizeof (*data)); + crctx->data = (void *) data; + quicly_ctx = &data->quicly_ctx; + ptls_ctx = &data->ptls_ctx; - quicly_ctx_data_t *quicly_ctx_data = - clib_mem_alloc (sizeof (quicly_ctx_data_t)); - clib_memset (quicly_ctx_data, 0, sizeof (*quicly_ctx_data)); /* picotls depends on this */ - quicly_ctx = &quicly_ctx_data->quicly_ctx; - ptls_context_t *ptls_ctx = &quicly_ctx_data->ptls_ctx; ptls_ctx->random_bytes = ptls_openssl_random_bytes; ptls_ctx->get_time = &ptls_get_time; ptls_ctx->key_exchanges = ptls_openssl_key_exchanges; - ptls_ctx->cipher_suites = qm->quic_ciphers[crypto_engine]; + ptls_ctx->cipher_suites = qm->quic_ciphers[ctx->crypto_engine]; ptls_ctx->certificates.list = NULL; ptls_ctx->certificates.count = 0; ptls_ctx->esni = NULL; @@ -91,58 +259,113 @@ quic_store_quicly_ctx (application_t * app, u32 ckpair_index, ptls_ctx->hkdf_label_prefix__obsolete = NULL; ptls_ctx->require_dhe_on_psk = 1; ptls_ctx->encrypt_ticket = &qm->session_cache.super; - - app->quicly_ctx = (u64 *) quicly_ctx; clib_memcpy (quicly_ctx, &quicly_spec_context, sizeof (quicly_context_t)); quicly_ctx->max_packet_size = QUIC_MAX_PACKET_SIZE; + quicly_ctx->max_packets_per_key = qm->max_packets_per_key; quicly_ctx->tls = ptls_ctx; quicly_ctx->stream_open = &on_stream_open; quicly_ctx->closed_by_peer = &on_closed_by_peer; quicly_ctx->now = &quicly_vpp_now_cb; quicly_amend_ptls_context (quicly_ctx->tls); + quicly_ctx->packet_allocator = &quic_packet_allocator; + quicly_ctx->crypto_engine = &quic_crypto_engine; quicly_ctx->transport_params.max_data = QUIC_INT_MAX; quicly_ctx->transport_params.max_streams_uni = (uint64_t) 1 << 60; quicly_ctx->transport_params.max_streams_bidi = (uint64_t) 1 << 60; - quicly_ctx->transport_params.idle_timeout = qm->connection_timeout; + quicly_ctx->transport_params.max_idle_timeout = qm->connection_timeout; - /* max_enq is FIFO_SIZE - 1 */ - max_enq = app->sm_properties.rx_fifo_size - 1; - quicly_ctx->transport_params.max_stream_data.bidi_local = max_enq; - max_enq = app->sm_properties.tx_fifo_size - 1; - quicly_ctx->transport_params.max_stream_data.bidi_remote = max_enq; + app = application_get (ctx->parent_app_id); + quicly_ctx->transport_params.max_stream_data.bidi_local = + app->sm_properties.rx_fifo_size - 1; + quicly_ctx->transport_params.max_stream_data.bidi_remote = + app->sm_properties.tx_fifo_size - 1; quicly_ctx->transport_params.max_stream_data.uni = QUIC_INT_MAX; - quicly_ctx->tls->random_bytes (quicly_ctx_data->cid_key, 16); - quicly_ctx_data->cid_key[16] = 0; - key_vec = ptls_iovec_init (quicly_ctx_data->cid_key, - strlen (quicly_ctx_data->cid_key)); + if (!app->quic_iv_set) + { + ptls_openssl_random_bytes (app->quic_iv, QUIC_IV_LEN - 1); + app->quic_iv[QUIC_IV_LEN - 1] = 0; + app->quic_iv_set = 1; + } + + clib_memcpy (data->cid_key, app->quic_iv, QUIC_IV_LEN); + key_vec = ptls_iovec_init (data->cid_key, QUIC_IV_LEN); quicly_ctx->cid_encryptor = quicly_new_default_cid_encryptor (&ptls_openssl_bfecb, &ptls_openssl_aes128ecb, &ptls_openssl_sha256, key_vec); - ckpair = app_cert_key_pair_get_if_valid (ckpair_index); + ckpair = app_cert_key_pair_get_if_valid (crctx->ckpair_index); if (!ckpair || !ckpair->key || !ckpair->cert) { - QUIC_DBG (1, "Wrong ckpair id %d\n", ckpair_index); - goto error; + QUIC_DBG (1, "Wrong ckpair id %d\n", crctx->ckpair_index); + return -1; } if (load_bio_private_key (quicly_ctx->tls, (char *) ckpair->key)) { QUIC_DBG (1, "failed to read private key from app configuration\n"); - goto error; + return -1; } if (load_bio_certificate_chain (quicly_ctx->tls, (char *) ckpair->cert)) { QUIC_DBG (1, "failed to load certificate\n"); - goto error; + return -1; + } + return 0; + +} + +static int +quic_acquire_crypto_context (quic_ctx_t * ctx) +{ + quic_main_t *qm = &quic_main; + crypto_context_t *crctx; + clib_bihash_kv_24_8_t kv; + + if (ctx->crypto_engine == CRYPTO_ENGINE_NONE) + { + QUIC_DBG (2, "No crypto engine specified, using %d", + qm->default_crypto_engine); + ctx->crypto_engine = qm->default_crypto_engine; + } + if (!clib_bitmap_get (qm->available_crypto_engines, ctx->crypto_engine)) + { + QUIC_DBG (1, "Quic does not support crypto engine %d", + ctx->crypto_engine); + return VNET_API_ERROR_MISSING_CERT_KEY; } + + /* Check for exisiting crypto ctx */ + quic_crypto_context_make_key_from_ctx (&kv, ctx); + if (clib_bihash_search_24_8 + (&qm->wrk_ctx[ctx->c_thread_index].crypto_context_hash, &kv, &kv) == 0) + { + crctx = quic_crypto_context_get (kv.value, ctx->c_thread_index); + QUIC_DBG (2, "Found exisiting crypto context %d", kv.value); + ctx->crypto_context_index = kv.value; + crctx->n_subscribers++; + return 0; + } + + crctx = quic_crypto_context_alloc (ctx->c_thread_index); + ctx->crypto_context_index = crctx->ctx_index; + kv.value = crctx->ctx_index; + crctx->crypto_engine = ctx->crypto_engine; + crctx->ckpair_index = ctx->ckpair_index; + if (quic_init_crypto_context (crctx, ctx)) + goto error; + if (vnet_app_add_cert_key_interest (ctx->ckpair_index, qm->app_index)) + goto error; + crctx->n_subscribers++; + clib_bihash_add_del_24_8 (&qm-> + wrk_ctx[ctx->c_thread_index].crypto_context_hash, + &kv, 1 /* is_add */ ); return 0; error: - clib_mem_free (quicly_ctx_data); + quic_crypto_context_free_if_needed (crctx, ctx->c_thread_index); return VNET_API_ERROR_MISSING_CERT_KEY; } @@ -169,7 +392,7 @@ quic_ctx_free (quic_ctx_t * ctx) { QUIC_DBG (2, "Free ctx %u %x", ctx->c_thread_index, ctx->c_c_index); u32 thread_index = ctx->c_thread_index; - ASSERT (ctx->timer_handle == QUIC_TIMER_HANDLE_INVALID); + QUIC_ASSERT (ctx->timer_handle == QUIC_TIMER_HANDLE_INVALID); if (CLIB_DEBUG) clib_memset (ctx, 0xfb, sizeof (*ctx)); pool_put (quic_main.ctx_pool[thread_index], ctx); @@ -189,7 +412,7 @@ quic_ctx_get_if_valid (u32 ctx_index, u32 thread_index) return pool_elt_at_index (quic_main.ctx_pool[thread_index], ctx_index); } -static quic_ctx_t * +quic_ctx_t * quic_get_conn_ctx (quicly_conn_t * conn) { u64 conn_data; @@ -216,6 +439,12 @@ quic_ctx_is_listener (quic_ctx_t * ctx) return (ctx->flags & QUIC_F_IS_LISTENER); } +static inline int +quic_ctx_is_conn (quic_ctx_t * ctx) +{ + return !(quic_ctx_is_listener (ctx) || quic_ctx_is_stream (ctx)); +} + static session_t * get_stream_session_from_stream (quicly_stream_t * stream) { @@ -247,7 +476,11 @@ quic_sendable_packet_count (session_t * udp_session) static quicly_context_t * quic_get_quicly_ctx_from_ctx (quic_ctx_t * ctx) { - return ctx->quicly_ctx; + crypto_context_t *crctx = + quic_crypto_context_get (ctx->crypto_context_index, ctx->c_thread_index); + quic_crypto_context_data_t *data = + (quic_crypto_context_data_t *) crctx->data; + return &data->quicly_ctx; } static quicly_context_t * @@ -256,7 +489,7 @@ quic_get_quicly_ctx_from_udp (u64 udp_session_handle) session_t *udp_session = session_get_from_handle (udp_session_handle); quic_ctx_t *ctx = quic_ctx_get (udp_session->opaque, udp_session->thread_index); - return ctx->quicly_ctx; + return quic_get_quicly_ctx_from_ctx (ctx); } static inline void @@ -295,14 +528,14 @@ quic_ack_rx_data (session_t * stream_session) sctx = quic_ctx_get (stream_session->connection_index, stream_session->thread_index); - ASSERT (quic_ctx_is_stream (sctx)); + QUIC_ASSERT (quic_ctx_is_stream (sctx)); stream = sctx->stream; stream_data = (quic_stream_data_t *) stream->data; f = stream_session->rx_fifo; max_deq = svm_fifo_max_dequeue (f); - ASSERT (stream_data->app_rx_data_len >= max_deq); + QUIC_ASSERT (stream_data->app_rx_data_len >= max_deq); quicly_stream_sync_recvbuf (stream, stream_data->app_rx_data_len - max_deq); QUIC_DBG (3, "Acking %u bytes", stream_data->app_rx_data_len - max_deq); stream_data->app_rx_data_len = max_deq; @@ -330,11 +563,12 @@ quic_connection_delete (quic_ctx_t * ctx) QUIC_DBG (2, "Deleting connection %u", ctx->c_c_index); - ASSERT (!quic_ctx_is_stream (ctx)); + QUIC_ASSERT (!quic_ctx_is_stream (ctx)); quic_stop_ctx_timer (ctx); /* Delete the connection from the connection map */ conn = ctx->conn; + ctx->conn = NULL; quic_make_connection_key (&kv, quicly_get_master_id (conn)); QUIC_DBG (2, "Deleting conn with id %lu %lu from map", kv.key[0], kv.key[1]); @@ -344,7 +578,6 @@ quic_connection_delete (quic_ctx_t * ctx) if (ctx->conn) quicly_free (ctx->conn); - ctx->conn = NULL; session_transport_delete_notify (&ctx->connection); } @@ -428,14 +661,14 @@ quic_send_datagram (session_t * udp_session, quicly_datagram_t * packet) /* Read dest address from quicly-provided sockaddr */ if (hdr.is_ip4) { - ASSERT (packet->dest.sa.sa_family == AF_INET); + QUIC_ASSERT (packet->dest.sa.sa_family == AF_INET); struct sockaddr_in *sa4 = (struct sockaddr_in *) &packet->dest.sa; hdr.rmt_port = sa4->sin_port; hdr.rmt_ip.ip4.as_u32 = sa4->sin_addr.s_addr; } else { - ASSERT (packet->dest.sa.sa_family == AF_INET6); + QUIC_ASSERT (packet->dest.sa.sa_family == AF_INET6); struct sockaddr_in6 *sa6 = (struct sockaddr_in6 *) &packet->dest.sa; hdr.rmt_port = sa6->sin6_port; clib_memcpy (&hdr.rmt_ip.ip6, &sa6->sin6_addr, 16); @@ -462,19 +695,20 @@ quic_send_datagram (session_t * udp_session, quicly_datagram_t * packet) static int quic_send_packets (quic_ctx_t * ctx) { + quic_main_t *qm = &quic_main; quicly_datagram_t *packets[QUIC_SEND_PACKET_VEC_SIZE]; session_t *udp_session; quicly_conn_t *conn; size_t num_packets, i, max_packets; quicly_packet_allocator_t *pa; - quicly_context_t *quicly_context; int err = 0; + u32 thread_index = vlib_get_thread_index (); /* We have sctx, get qctx */ if (quic_ctx_is_stream (ctx)) ctx = quic_ctx_get (ctx->quic_connection_ctx_id, ctx->c_thread_index); - ASSERT (!quic_ctx_is_stream (ctx)); + QUIC_ASSERT (!quic_ctx_is_stream (ctx)); udp_session = session_get_from_handle_if_valid (ctx->udp_session_handle); if (!udp_session) @@ -489,15 +723,7 @@ quic_send_packets (quic_ctx_t * ctx) if (quic_sendable_packet_count (udp_session) < 2) goto stop_sending; - quicly_context = quic_get_quicly_ctx_from_ctx (ctx); - if (!quicly_context) - { - clib_warning ("Tried to send packets on non existing app worker %u", - ctx->parent_app_wrk_id); - quic_connection_delete (ctx); - return 1; - } - pa = quicly_context->packet_allocator; + pa = quic_get_quicly_ctx_from_ctx (ctx)->packet_allocator; do { max_packets = quic_sendable_packet_count (udp_session); @@ -507,8 +733,12 @@ quic_send_packets (quic_ctx_t * ctx) if ((err = quicly_send (conn, packets, &num_packets))) goto quicly_error; + quic_crypto_batch_tx_packets (&qm->wrk_ctx + [thread_index].crypto_context_batch); + for (i = 0; i != num_packets; ++i) { + quic_crypto_finalize_send_packet (packets[i]); if ((err = quic_send_datagram (udp_session, packets[i]))) goto quicly_error; @@ -549,6 +779,7 @@ quic_on_stream_destroy (quicly_stream_t * stream, int err) stream_session->session_state = SESSION_STATE_CLOSED; session_transport_delete_notify (&sctx->connection); + quic_increment_counter (QUIC_ERROR_CLOSED_STREAM, 1); quic_ctx_free (sctx); clib_mem_free (stream->data); } @@ -590,13 +821,12 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, size_t len) { QUIC_DBG (3, "received data: %lu bytes, offset %lu", len, off); - u32 max_enq; + u32 max_enq, rlen, rv; quic_ctx_t *sctx; session_t *stream_session; app_worker_t *app_wrk; svm_fifo_t *f; quic_stream_data_t *stream_data; - int rlen, rv; stream_data = (quic_stream_data_t *) stream->data; sctx = quic_ctx_get (stream_data->ctx_id, stream_data->thread_index); @@ -640,7 +870,7 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, stream_session->app_wrk_index, stream_session->thread_index, f, len, rlen, off, max_enq); stream_data->app_rx_data_len += rlen; - ASSERT (rlen >= len); + QUIC_ASSERT (rlen >= len); app_wrk = app_worker_get_if_valid (stream_session->app_wrk_index); if (PREDICT_TRUE (app_wrk != 0)) { @@ -656,7 +886,7 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, rlen = svm_fifo_enqueue_with_offset (f, off - stream_data->app_rx_data_len, len, (u8 *) src); - ASSERT (rlen == 0); + QUIC_ASSERT (rlen == 0); } return 0; } @@ -664,33 +894,41 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, void quic_fifo_egress_shift (quicly_stream_t * stream, size_t delta) { + quic_stream_data_t *stream_data; session_t *stream_session; svm_fifo_t *f; - int rv; + u32 rv; + stream_data = (quic_stream_data_t *) stream->data; stream_session = get_stream_session_from_stream (stream); f = stream_session->tx_fifo; + QUIC_ASSERT (stream_data->app_tx_data_len >= delta); + stream_data->app_tx_data_len -= delta; rv = svm_fifo_dequeue_drop (f, delta); - ASSERT (rv == delta); - quicly_stream_sync_sendbuf (stream, 0); + QUIC_ASSERT (rv == delta); + + rv = quicly_stream_sync_sendbuf (stream, 0); + QUIC_ASSERT (!rv); } int quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst, size_t * len, int *wrote_all) { - u32 deq_max, first_deq, max_rd_chunk, rem_offset; + quic_stream_data_t *stream_data; session_t *stream_session; svm_fifo_t *f; + u32 deq_max; + stream_data = (quic_stream_data_t *) stream->data; stream_session = get_stream_session_from_stream (stream); f = stream_session->tx_fifo; QUIC_DBG (3, "Emitting %u, offset %u", *len, off); deq_max = svm_fifo_max_dequeue_cons (f); - ASSERT (off <= deq_max); + QUIC_ASSERT (off <= deq_max); if (off + *len < deq_max) { *wrote_all = 0; @@ -699,25 +937,13 @@ quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst, { *wrote_all = 1; *len = deq_max - off; - QUIC_DBG (3, "Wrote ALL, %u", *len); } + QUIC_ASSERT (*len > 0); - /* TODO, use something like : return svm_fifo_peek (f, off, *len, dst); */ - max_rd_chunk = svm_fifo_max_read_chunk (f); - - first_deq = 0; - if (off < max_rd_chunk) - { - first_deq = clib_min (*len, max_rd_chunk - off); - clib_memcpy_fast (dst, svm_fifo_head (f) + off, first_deq); - } + if (off + *len > stream_data->app_tx_data_len) + stream_data->app_tx_data_len = off + *len; - if (max_rd_chunk < off + *len) - { - rem_offset = max_rd_chunk < off ? off - max_rd_chunk : 0; - clib_memcpy_fast (dst + first_deq, f->head_chunk->data + rem_offset, - *len - first_deq); - } + svm_fifo_peek (f, off, *len, dst); return 0; } @@ -775,11 +1001,14 @@ quic_on_stream_open (quicly_stream_open_t * self, quicly_stream_t * stream) sctx->stream = stream; sctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; sctx->flags |= QUIC_F_IS_STREAM; + if (quicly_stream_is_unidirectional (stream->stream_id)) + stream_session->flags |= SESSION_F_UNIDIRECTIONAL; stream_data = (quic_stream_data_t *) stream->data; stream_data->ctx_id = sctx_id; stream_data->thread_index = sctx->c_thread_index; stream_data->app_rx_data_len = 0; + stream_data->app_tx_data_len = 0; sctx->c_s_index = stream_session->session_index; stream_session->session_state = SESSION_STATE_CREATED; @@ -946,9 +1175,8 @@ quic_expired_timers_dispatch (u32 * expired_timers) } /* Transport proto functions */ - static int -quic_connect_stream (session_t * quic_session, u32 opaque) +quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) { uint64_t quic_session_handle; session_t *stream_session; @@ -1001,11 +1229,15 @@ quic_connect_stream (session_t * quic_session, u32 opaque) if (!conn || !quicly_connection_is_ready (conn)) return -1; - if ((rv = quicly_open_stream (conn, &stream, 0 /* uni */ ))) + if ((rv = + quicly_open_stream (conn, &stream, + sep->flags & SESSION_F_UNIDIRECTIONAL))) { QUIC_DBG (2, "Stream open failed with %d", rv); return -1; } + quic_increment_counter (QUIC_ERROR_OPENED_STREAM, 1); + sctx->stream = stream; QUIC_DBG (2, "Opened stream %d, creating session", stream->stream_id); @@ -1018,12 +1250,15 @@ quic_connect_stream (session_t * quic_session, u32 opaque) stream_session->listener_handle = quic_session_handle; stream_session->session_type = session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, qctx->udp_is_ip4); + if (sep->flags & SESSION_F_UNIDIRECTIONAL) + stream_session->flags |= SESSION_F_UNIDIRECTIONAL; sctx->c_s_index = stream_session->session_index; stream_data = (quic_stream_data_t *) stream->data; stream_data->ctx_id = sctx->c_c_index; stream_data->thread_index = sctx->c_thread_index; stream_data->app_rx_data_len = 0; + stream_data->app_tx_data_len = 0; stream_session->session_state = SESSION_STATE_READY; /* For now we only reset streams. Cleanup will be triggered by timers */ @@ -1031,16 +1266,17 @@ quic_connect_stream (session_t * quic_session, u32 opaque) { QUIC_ERR ("failed to app_worker_init_connected"); quicly_reset_stream (stream, QUIC_APP_CONNECT_NOTIFY_ERROR); - return app_worker_connect_notify (app_wrk, NULL, opaque); + return app_worker_connect_notify (app_wrk, NULL, sep->opaque); } svm_fifo_add_want_deq_ntf (stream_session->rx_fifo, SVM_FIFO_WANT_DEQ_NOTIF_IF_FULL | SVM_FIFO_WANT_DEQ_NOTIF_IF_EMPTY); - if (app_worker_connect_notify (app_wrk, stream_session, opaque)) + if (app_worker_connect_notify (app_wrk, stream_session, sep->opaque)) { QUIC_ERR ("failed to notify app"); + quic_increment_counter (QUIC_ERROR_CLOSED_STREAM, 1); quicly_reset_stream (stream, QUIC_APP_CONNECT_NOTIFY_ERROR); return -1; } @@ -1089,12 +1325,10 @@ quic_connect_connection (session_endpoint_cfg_t * sep) ctx->parent_app_id = app_wrk->app_index; cargs->sep_ext.ns_index = app->ns_index; - if ((error = - quic_store_quicly_ctx (app, sep->ckpair_index, sep->crypto_engine))) + ctx->crypto_engine = sep->crypto_engine; + ctx->ckpair_index = sep->ckpair_index; + if ((error = quic_acquire_crypto_context (ctx))) return error; - /* Also store it in ctx for convenience - * Waiting for crypto_ctx logic */ - ctx->quicly_ctx = (quicly_context_t *) app->quicly_ctx; if ((error = vnet_connect (cargs))) return error; @@ -1112,7 +1346,7 @@ quic_connect (transport_endpoint_cfg_t * tep) quic_session = session_get_from_handle_if_valid (sep->parent_handle); if (quic_session) - return quic_connect_stream (quic_session, sep->opaque); + return quic_connect_stream (quic_session, sep); else return quic_connect_connection (sep); } @@ -1131,6 +1365,9 @@ quic_proto_on_close (u32 ctx_index, u32 thread_index) if (quic_ctx_is_stream (ctx)) { quicly_stream_t *stream = ctx->stream; + if (!quicly_stream_has_send_side (quicly_is_client (stream->conn), + stream->stream_id)) + return; quicly_reset_stream (stream, QUIC_APP_ERROR_CLOSE_NOTIFY); quic_send_packets (ctx); return; @@ -1145,6 +1382,8 @@ quic_proto_on_close (u32 ctx_index, u32 thread_index) quicly_conn_t *conn = ctx->conn; /* Start connection closing. Keep sending packets until quicly_send returns QUICLY_ERROR_FREE_CONNECTION */ + + quic_increment_counter (QUIC_ERROR_CLOSED_CONNECTION, 1); quicly_close (conn, QUIC_APP_ERROR_CLOSE_NOTIFY, "Closed by peer"); /* This also causes all streams to be closed (and the cb called) */ quic_send_packets (ctx); @@ -1188,14 +1427,11 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) app = application_get (app_wrk->app_index); QUIC_DBG (2, "Called quic_start_listen for app %d", app_wrk->app_index); - if (quic_store_quicly_ctx (app, sep->ckpair_index, sep->crypto_engine)) - return -1; - - sep->transport_proto = TRANSPORT_PROTO_UDPC; clib_memset (args, 0, sizeof (*args)); args->app_index = qm->app_index; args->sep_ext = *sep; args->sep_ext.ns_index = app->ns_index; + args->sep_ext.transport_proto = TRANSPORT_PROTO_UDPC; if ((rv = vnet_listen (args))) return rv; @@ -1207,9 +1443,6 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) lctx = quic_ctx_get (lctx_index, 0); lctx->flags |= QUIC_F_IS_LISTENER; - /* Also store it in ctx for convenience - * Waiting for crypto_ctx logic */ - lctx->quicly_ctx = (quicly_context_t *) app->quicly_ctx; clib_memcpy (&lctx->c_rmt_ip, &args->sep.peer.ip, sizeof (ip46_address_t)); clib_memcpy (&lctx->c_lcl_ip, &args->sep.ip, sizeof (ip46_address_t)); @@ -1222,6 +1455,10 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) lctx->parent_app_id = app_wrk->app_index; lctx->udp_session_handle = udp_handle; lctx->c_s_index = quic_listen_session_index; + lctx->crypto_engine = sep->crypto_engine; + lctx->ckpair_index = sep->ckpair_index; + if (quic_acquire_crypto_context (lctx)) + return -1; QUIC_DBG (2, "Listening UDP session 0x%lx", session_handle (udp_listen_session)); @@ -1235,7 +1472,7 @@ quic_stop_listen (u32 lctx_index) QUIC_DBG (2, "Called quic_stop_listen"); quic_ctx_t *lctx; lctx = quic_ctx_get (lctx_index, 0); - ASSERT (quic_ctx_is_listener (lctx)); + QUIC_ASSERT (quic_ctx_is_listener (lctx)); vnet_unlisten_args_t a = { .handle = lctx->udp_session_handle, .app_index = quic_main.app_index, @@ -1244,8 +1481,8 @@ quic_stop_listen (u32 lctx_index) if (vnet_unlisten (&a)) clib_warning ("unlisten errored"); - /* TODO: crypto state cleanup */ - + quic_release_crypto_context (lctx->crypto_context_index, + 0 /* thread_index */ ); quic_ctx_free (lctx); return 0; } @@ -1422,6 +1659,19 @@ quic_check_quic_session_connected (quic_ctx_t * ctx) quic_on_quic_session_connected (ctx); } +static inline void +quic_update_conn_ctx (quicly_conn_t * conn, quicly_context_t * quicly_context) +{ + /* we need to update the quicly_conn on migrate + * as it contains a pointer to the crypto context */ + ptls_context_t **tls; + quicly_context_t **_quicly_context; + _quicly_context = (quicly_context_t **) conn; + *_quicly_context = quicly_context; + tls = (ptls_context_t **) quicly_get_tls (conn); + *tls = quicly_context->tls; +} + static void quic_receive_connection (void *arg) { @@ -1429,6 +1679,7 @@ quic_receive_connection (void *arg) quic_ctx_t *temp_ctx, *new_ctx; clib_bihash_kv_16_8_t kv; quicly_conn_t *conn; + quicly_context_t *quicly_context; session_t *udp_session; temp_ctx = arg; @@ -1443,8 +1694,12 @@ quic_receive_connection (void *arg) new_ctx->c_thread_index = thread_index; new_ctx->c_c_index = new_ctx_id; + quic_acquire_crypto_context (new_ctx); conn = new_ctx->conn; + quicly_context = quic_get_quicly_ctx_from_ctx (new_ctx); + quic_update_conn_ctx (conn, quicly_context); + quic_store_conn_ctx (conn, new_ctx); quic_make_connection_key (&kv, quicly_get_master_id (conn)); kv.value = ((u64) thread_index) << 32 | (u64) new_ctx_id; @@ -1470,12 +1725,13 @@ quic_transfer_connection (u32 ctx_index, u32 dest_thread) QUIC_DBG (2, "Transferring conn %u to thread %u", ctx_index, dest_thread); temp_ctx = clib_mem_alloc (sizeof (quic_ctx_t)); - ASSERT (temp_ctx); + QUIC_ASSERT (temp_ctx != NULL); ctx = quic_ctx_get (ctx_index, thread_index); clib_memcpy (temp_ctx, ctx, sizeof (quic_ctx_t)); quic_stop_ctx_timer (ctx); + quic_release_crypto_context (ctx->crypto_context_index, thread_index); quic_ctx_free (ctx); /* Send connection to destination thread */ @@ -1533,10 +1789,10 @@ quic_udp_session_connected_callback (u32 quic_app_index, u32 ctx_index, quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx); ret = quicly_connect (&ctx->conn, quicly_ctx, (char *) ctx->srv_hostname, - sa, NULL, &quic_main.next_cid, ptls_iovec_init (NULL, - 0), - &quic_main.hs_properties, NULL); - ++quic_main.next_cid.master_id; + sa, NULL, &quic_main.wrk_ctx[thread_index].next_cid, + ptls_iovec_init (NULL, 0), &quic_main.hs_properties, + NULL); + ++quic_main.wrk_ctx[thread_index].next_cid.master_id; /* Save context handle in quicly connection */ quic_store_conn_ctx (ctx->conn, ctx); assert (ret == 0); @@ -1574,6 +1830,8 @@ quic_udp_session_cleanup_callback (session_t * udp_session, ctx = quic_ctx_get (udp_session->opaque, udp_session->thread_index); quic_stop_ctx_timer (ctx); + quic_release_crypto_context (ctx->crypto_context_index, + ctx->c_thread_index); quic_ctx_free (ctx); } @@ -1589,10 +1847,10 @@ quic_udp_session_migrate_callback (session_t * s, session_handle_t new_sh) u32 new_thread = session_thread_from_handle (new_sh); quic_ctx_t *ctx; - QUIC_ERR ("Session %x migrated to %lx", s->session_index, new_sh); - ASSERT (vlib_get_thread_index () == s->thread_index); + QUIC_DBG (2, "Session %x migrated to %lx", s->session_index, new_sh); + QUIC_ASSERT (vlib_get_thread_index () == s->thread_index); ctx = quic_ctx_get (s->opaque, s->thread_index); - ASSERT (ctx->udp_session_handle == session_handle (s)); + QUIC_ASSERT (ctx->udp_session_handle == session_handle (s)); ctx->udp_session_handle = new_sh; #if QUIC_DEBUG >= 1 @@ -1630,10 +1888,9 @@ quic_udp_session_accepted_callback (session_t * udp_session) ctx->conn_state = QUIC_CONN_STATE_OPENED; ctx->c_flags |= TRANSPORT_CONNECTION_F_NO_LOOKUP; - /* Also store it in ctx for convenience - * Waiting for crypto_ctx logic */ - ctx->quicly_ctx = lctx->quicly_ctx; - + ctx->crypto_engine = lctx->crypto_engine; + ctx->ckpair_index = lctx->ckpair_index; + quic_acquire_crypto_context (ctx); udp_session->opaque = ctx_index; /* TODO timeout to delete these if they never connect */ @@ -1674,8 +1931,10 @@ static int quic_custom_tx_callback (void *s, u32 max_burst_size) { session_t *stream_session = (session_t *) s; + quic_stream_data_t *stream_data; quicly_stream_t *stream; quic_ctx_t *ctx; + u32 max_deq; int rv; if (PREDICT_FALSE @@ -1690,9 +1949,6 @@ quic_custom_tx_callback (void *s, u32 max_burst_size) QUIC_DBG (3, "Stream TX event"); quic_ack_rx_data (stream_session); - if (!svm_fifo_max_dequeue (stream_session->tx_fifo)) - return 0; - stream = ctx->stream; if (!quicly_sendstate_is_open (&stream->sendstate)) { @@ -1700,8 +1956,18 @@ quic_custom_tx_callback (void *s, u32 max_burst_size) return -1; } - if ((rv = quicly_stream_sync_sendbuf (stream, 1)) != 0) - return rv; + stream_data = (quic_stream_data_t *) stream->data; + max_deq = svm_fifo_max_dequeue (stream_session->tx_fifo); + QUIC_ASSERT (max_deq >= stream_data->app_tx_data_len); + if (max_deq == stream_data->app_tx_data_len) + { + QUIC_DBG (3, "TX but no data %d / %d", max_deq, + stream_data->app_tx_data_len); + return 0; + } + stream_data->app_tx_data_len = max_deq; + rv = quicly_stream_sync_sendbuf (stream, 1); + QUIC_ASSERT (!rv); tx_end: quic_send_packets (ctx); @@ -1759,10 +2025,9 @@ quic_find_packet_ctx (quic_rx_packet_ctx_t * pctx, u32 caller_thread_index) return QUIC_PACKET_TYPE_RECEIVE; } -static int -quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) +static void +quic_accept_connection (quic_rx_packet_ctx_t * pctx) { - u32 thread_index = vlib_get_thread_index (); quicly_context_t *quicly_ctx; session_t *quic_session; clib_bihash_kv_16_8_t kv; @@ -1774,26 +2039,28 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) /* new connection, accept and create context if packet is valid * TODO: check if socket is actually listening? */ - ctx = quic_ctx_get (ctx_index, thread_index); + ctx = quic_ctx_get (pctx->ctx_index, pctx->thread_index); if (ctx->c_s_index != QUIC_SESSION_INVALID) { - QUIC_DBG (2, "already accepted ctx 0x%x", ctx_index); - return 0; + QUIC_DBG (2, "already accepted ctx 0x%x", ctx->c_s_index); + return; } quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx); if ((rv = quicly_accept (&conn, quicly_ctx, NULL, &pctx->sa, - &pctx->packet, NULL, &quic_main.next_cid, NULL))) + &pctx->packet, NULL, + &quic_main.wrk_ctx[pctx->thread_index].next_cid, + NULL))) { /* Invalid packet, pass */ assert (conn == NULL); QUIC_ERR ("Accept failed with %U", quic_format_err, rv); /* TODO: cleanup created quic ctx and UDP session */ - return 0; + return; } assert (conn != NULL); - ++quic_main.next_cid.master_id; + ++quic_main.wrk_ctx[pctx->thread_index].next_cid.master_id; /* Save ctx handle in quicly connection */ quic_store_conn_ctx (conn, ctx); ctx->conn = conn; @@ -1814,7 +2081,7 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) /* Register connection in connections map */ quic_make_connection_key (&kv, quicly_get_master_id (conn)); - kv.value = ((u64) thread_index) << 32 | (u64) ctx_index; + kv.value = ((u64) pctx->thread_index) << 32 | (u64) pctx->ctx_index; clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 1 /* is_add */ ); QUIC_DBG (2, "Registering conn with id %lu %lu", kv.key[0], kv.key[1]); @@ -1822,20 +2089,19 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) if ((rv = app_worker_init_accepted (quic_session))) { QUIC_ERR ("failed to allocate fifos"); - quic_proto_on_close (ctx_index, thread_index); - return rv; + quic_proto_on_close (pctx->ctx_index, pctx->thread_index); + return; } app_wrk = app_worker_get (quic_session->app_wrk_index); if ((rv = app_worker_accept_notify (app_wrk, quic_session))) { QUIC_ERR ("failed to notify accept worker app"); - quic_proto_on_close (ctx_index, thread_index); - return rv; + quic_proto_on_close (pctx->ctx_index, pctx->thread_index); + return; } - ctx->conn_state = QUIC_CONN_STATE_HANDSHAKE; - return quic_send_packets (ctx); + ctx->conn_state = QUIC_CONN_STATE_READY; } static int @@ -1874,12 +2140,13 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, u32 thread_index = vlib_get_thread_index (); u32 cur_deq = svm_fifo_max_dequeue (f) - fifo_offset; quicly_context_t *quicly_ctx; + session_t *udp_session; int rv; ret = svm_fifo_peek (f, fifo_offset, SESSION_CONN_HDR_LEN, (u8 *) & pctx->ph); - ASSERT (ret == SESSION_CONN_HDR_LEN); - ASSERT (pctx->ph.data_offset == 0); + QUIC_ASSERT (ret == SESSION_CONN_HDR_LEN); + QUIC_ASSERT (pctx->ph.data_offset == 0); full_len = pctx->ph.data_length + SESSION_CONN_HDR_LEN; if (full_len > cur_deq) { @@ -1913,6 +2180,11 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, if (rv == QUIC_PACKET_TYPE_RECEIVE) { pctx->ptype = QUIC_PACKET_TYPE_RECEIVE; + if (quic_main.vnet_crypto_enabled) + { + quic_ctx_t *qctx = quic_ctx_get (pctx->ctx_index, thread_index); + quic_crypto_decrypt_packet (qctx, pctx); + } return 0; } else if (rv == QUIC_PACKET_TYPE_MIGRATE) @@ -1923,6 +2195,9 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, else if (QUICLY_PACKET_IS_LONG_HEADER (pctx->packet.octets.base[0])) { pctx->ptype = QUIC_PACKET_TYPE_ACCEPT; + udp_session = session_get_from_handle (udp_session_handle); + pctx->ctx_index = udp_session->opaque; + pctx->thread_index = thread_index; } else { @@ -1935,7 +2210,8 @@ static int quic_udp_session_rx_callback (session_t * udp_session) { /* Read data from UDP rx_fifo and pass it to the quicly conn. */ - quic_ctx_t *ctx = NULL; + quic_main_t *qm = &quic_main; + quic_ctx_t *ctx = NULL, *prev_ctx = NULL; svm_fifo_t *f = udp_session->rx_fifo; u32 max_deq; u64 udp_session_handle = session_handle (udp_session); @@ -1959,6 +2235,11 @@ rx_start: fifo_offset = 0; max_packets = QUIC_RCV_MAX_BATCH_PACKETS; +#if CLIB_DEBUG > 0 + clib_memset (packets_ctx, 0xfa, + QUIC_RCV_MAX_BATCH_PACKETS * sizeof (quic_rx_packet_ctx_t)); +#endif + for (i = 0; i < max_packets; i++) { packets_ctx[i].thread_index = UINT32_MAX; @@ -1989,6 +2270,9 @@ rx_start: } } + quic_crypto_batch_rx_packets (&qm-> + wrk_ctx[thread_index].crypto_context_batch); + for (i = 0; i < max_packets; i++) { switch (packets_ctx[i].ptype) @@ -2004,28 +2288,36 @@ rx_start: } break; case QUIC_PACKET_TYPE_ACCEPT: - udp_session = session_get_from_handle (udp_session_handle); - if ((rv = quic_accept_connection (udp_session->opaque, - &packets_ctx[i]))) - { - QUIC_ERR ("quic accept errored with %d", rv); - } + quic_accept_connection (&packets_ctx[i]); break; case QUIC_PACKET_TYPE_RESET: quic_reset_connection (udp_session_handle, &packets_ctx[i]); break; } } + ctx = prev_ctx = NULL; for (i = 0; i < max_packets; i++) { - if (packets_ctx[i].ptype != QUIC_PACKET_TYPE_RECEIVE) - continue; - ctx = quic_ctx_get (packets_ctx[i].ctx_index, - packets_ctx[i].thread_index); - quic_check_quic_session_connected (ctx); - ctx = quic_ctx_get (packets_ctx[i].ctx_index, - packets_ctx[i].thread_index); - quic_send_packets (ctx); + prev_ctx = ctx; + switch (packets_ctx[i].ptype) + { + case QUIC_PACKET_TYPE_RECEIVE: + ctx = quic_ctx_get (packets_ctx[i].ctx_index, + packets_ctx[i].thread_index); + quic_check_quic_session_connected (ctx); + ctx = quic_ctx_get (packets_ctx[i].ctx_index, + packets_ctx[i].thread_index); + break; + case QUIC_PACKET_TYPE_ACCEPT: + ctx = quic_ctx_get (packets_ctx[i].ctx_index, + packets_ctx[i].thread_index); + break; + default: + continue; /* this exits the for loop since other packet types are + necessarily the last in the batch */ + } + if (ctx != prev_ctx) + quic_send_packets (ctx); } udp_session = session_get_from_handle (udp_session_handle); /* session alloc might have happened */ @@ -2087,6 +2379,7 @@ static session_cb_vft_t quic_app_cb_vft = { .del_segment_callback = quic_del_segment_callback, .builtin_app_rx_callback = quic_udp_session_rx_callback, .session_cleanup_callback = quic_udp_session_cleanup_callback, + .app_cert_key_pair_delete_callback = quic_app_cert_key_pair_delete_callback, }; static const transport_proto_vft_t quic_proto = { @@ -2105,6 +2398,8 @@ static const transport_proto_vft_t quic_proto = { .get_transport_endpoint = quic_get_transport_endpoint, .get_transport_listener_endpoint = quic_get_transport_listener_endpoint, .transport_options = { + .name = "quic", + .short_name = "Q", .tx_type = TRANSPORT_TX_INTERNAL, .service_type = TRANSPORT_SERVICE_APP, }, @@ -2180,19 +2475,24 @@ quic_init (vlib_main_t * vm) vec_validate (qm->ctx_pool, num_threads - 1); vec_validate (qm->wrk_ctx, num_threads - 1); - /* Timer wheels, one per thread. */ + for (i = 0; i < num_threads; i++) { + qm->wrk_ctx[i].next_cid.thread_id = i; tw = &qm->wrk_ctx[i].timer_wheel; tw_timer_wheel_init_1t_3w_1024sl_ov (tw, quic_expired_timers_dispatch, 1e-3 /* timer period 1ms */ , ~0); tw->last_run_time = vlib_time_now (vlib_get_main ()); + clib_bihash_init_24_8 (&qm->wrk_ctx[i].crypto_context_hash, + "quic crypto contexts", 64, 128 << 10); + + qm->wrk_ctx[i].crypto_context_batch.nb_rx_packets = 0; + qm->wrk_ctx[i].crypto_context_batch.nb_tx_packets = 0; } clib_bihash_init_16_8 (&qm->connection_hash, "quic connections", 1024, 4 << 20); - qm->app_index = a->app_index; qm->tstamp_ticks_per_clock = vm->clib_time.seconds_per_clock / QUIC_TSTAMP_RESOLUTION; @@ -2208,7 +2508,16 @@ quic_init (vlib_main_t * vm) quic_register_cipher_suite (CRYPTO_ENGINE_VPP, quic_crypto_cipher_suites); quic_register_cipher_suite (CRYPTO_ENGINE_PICOTLS, ptls_openssl_cipher_suites); - qm->default_crypto_engine = CRYPTO_ENGINE_PICOTLS; + qm->default_crypto_engine = CRYPTO_ENGINE_VPP; + qm->max_packets_per_key = DEFAULT_MAX_PACKETS_PER_KEY; + clib_rwlock_init (&qm->crypto_keys_quic_rw_lock); + + vnet_crypto_main_t *cm = &crypto_main; + if (vec_len (cm->engines) == 0) + qm->vnet_crypto_enabled = 0; + else + qm->vnet_crypto_enabled = 1; + vec_free (a->name); return 0; } @@ -2267,65 +2576,239 @@ quic_plugin_set_fifo_size_command_fn (vlib_main_t * vm, return 0; } -static u8 * -quic_format_ctx_stat (u8 * s, va_list * args) +static inline u64 +quic_get_counter_value (u32 event_code) { - quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *); - quicly_stats_t quicly_stats; + vlib_node_t *n; + vlib_main_t *vm; + vlib_error_main_t *em; - quicly_get_stats (ctx->conn, &quicly_stats); + u32 code, i; + u64 c, sum = 0; + int index = 0; - s = format (s, "\n\rQUIC conn stats \n\r"); + vm = vlib_get_main (); + em = &vm->error_main; + n = vlib_get_node (vm, quic_input_node.index); + code = event_code; + /* *INDENT-OFF* */ + foreach_vlib_main(({ + em = &this_vlib_main->error_main; + i = n->error_heap_index + code; + c = em->counters[i]; - s = - format (s, "RTT: min:%d, smoothed:%d, variance:%d, latest:%d \n\r", - quicly_stats.rtt.minimum, quicly_stats.rtt.smoothed, - quicly_stats.rtt.variance, quicly_stats.rtt.latest); - s = format (s, "Packet loss:%d \n\r", quicly_stats.num_packets.lost); - - return s; + if (i < vec_len (em->counters_last_clear)) + c -= em->counters_last_clear[i]; + sum += c; + index++; + })); + /* *INDENT-ON* */ + return sum; } -static clib_error_t * -quic_plugin_showstats_command_fn (vlib_main_t * vm, - unformat_input_t * input, - vlib_cli_command_t * cmd) +static void +quic_show_aggregated_stats (vlib_main_t * vm) { + u32 num_workers = vlib_num_workers (); quic_main_t *qm = &quic_main; quic_ctx_t *ctx = NULL; - u32 num_workers = vlib_num_workers (); + quicly_stats_t st, agg_stats; + u32 i, nconn = 0, nstream = 0; - for (int i = 0; i < num_workers + 1; i++) + clib_memset (&agg_stats, 0, sizeof (agg_stats)); + for (i = 0; i < num_workers + 1; i++) { /* *INDENT-OFF* */ pool_foreach (ctx, qm->ctx_pool[i], ({ - if(!(ctx->flags & QUIC_F_IS_LISTENER) && !(ctx->flags & QUIC_F_IS_STREAM)) - vlib_cli_output (vm, "%U", quic_format_ctx_stat, ctx); + if (quic_ctx_is_conn (ctx) && ctx->conn) + { + quicly_get_stats (ctx->conn, &st); + agg_stats.rtt.smoothed += st.rtt.smoothed; + agg_stats.rtt.minimum += st.rtt.minimum; + agg_stats.rtt.variance += st.rtt.variance; + agg_stats.num_packets.received += st.num_packets.received; + agg_stats.num_packets.sent += st.num_packets.sent; + agg_stats.num_packets.lost += st.num_packets.lost; + agg_stats.num_packets.ack_received += st.num_packets.ack_received; + agg_stats.num_bytes.received += st.num_bytes.received; + agg_stats.num_bytes.sent += st.num_bytes.sent; + nconn++; + } + else if (quic_ctx_is_stream (ctx)) + nstream++; })); /* *INDENT-ON* */ } - return 0; + vlib_cli_output (vm, "-------- Connections --------"); + vlib_cli_output (vm, "Current: %u", nconn); + vlib_cli_output (vm, "Opened: %d", + quic_get_counter_value (QUIC_ERROR_OPENED_CONNECTION)); + vlib_cli_output (vm, "Closed: %d", + quic_get_counter_value (QUIC_ERROR_CLOSED_CONNECTION)); + vlib_cli_output (vm, "---------- Streams ----------"); + vlib_cli_output (vm, "Current: %u", nstream); + vlib_cli_output (vm, "Opened: %d", + quic_get_counter_value (QUIC_ERROR_OPENED_STREAM)); + vlib_cli_output (vm, "Closed: %d", + quic_get_counter_value (QUIC_ERROR_CLOSED_STREAM)); + vlib_cli_output (vm, "---------- Packets ----------"); + vlib_cli_output (vm, "RX Total: %d", + quic_get_counter_value (QUIC_ERROR_RX_PACKETS)); + vlib_cli_output (vm, "RX 0RTT: %d", + quic_get_counter_value (QUIC_ERROR_ZERO_RTT_RX_PACKETS)); + vlib_cli_output (vm, "RX 1RTT: %d", + quic_get_counter_value (QUIC_ERROR_ONE_RTT_RX_PACKETS)); + vlib_cli_output (vm, "TX Total: %d", + quic_get_counter_value (QUIC_ERROR_TX_PACKETS)); + vlib_cli_output (vm, "----------- Stats -----------"); + vlib_cli_output (vm, "Min RTT %f", + nconn > 0 ? agg_stats.rtt.minimum / nconn : 0); + vlib_cli_output (vm, "Smoothed RTT %f", + nconn > 0 ? agg_stats.rtt.smoothed / nconn : 0); + vlib_cli_output (vm, "Variance on RTT %f", + nconn > 0 ? agg_stats.rtt.variance / nconn : 0); + vlib_cli_output (vm, "Packets Received %lu", + agg_stats.num_packets.received); + vlib_cli_output (vm, "Packets Sent %lu", agg_stats.num_packets.sent); + vlib_cli_output (vm, "Packets Lost %lu", agg_stats.num_packets.lost); + vlib_cli_output (vm, "Packets Acks %lu", + agg_stats.num_packets.ack_received); + vlib_cli_output (vm, "RX bytes %lu", agg_stats.num_bytes.received); + vlib_cli_output (vm, "TX bytes %lu", agg_stats.num_bytes.sent); +} + +static u8 * +quic_format_quicly_conn_id (u8 * s, va_list * args) +{ + quicly_cid_plaintext_t *mid = va_arg (*args, quicly_cid_plaintext_t *); + s = format (s, "C%x_%x", mid->master_id, mid->thread_id); + return s; +} + +static u8 * +quic_format_quicly_stream_id (u8 * s, va_list * args) +{ + quicly_stream_t *stream = va_arg (*args, quicly_stream_t *); + s = + format (s, "%U S%lx", quic_format_quicly_conn_id, + quicly_get_master_id (stream->conn), stream->stream_id); + return s; +} + +static u8 * +quic_format_listener_ctx (u8 * s, va_list * args) +{ + quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *); + s = format (s, "[#%d][%x][Listener]", ctx->c_thread_index, ctx->c_c_index); + return s; +} + +static u8 * +quic_format_connection_ctx (u8 * s, va_list * args) +{ + quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *); + quicly_stats_t quicly_stats; + + s = format (s, "[#%d][%x]", ctx->c_thread_index, ctx->c_c_index); + + if (!ctx->conn) + { + s = format (s, "- no conn -\n"); + return s; + } + s = format (s, "[%U]", + quic_format_quicly_conn_id, quicly_get_master_id (ctx->conn)); + quicly_get_stats (ctx->conn, &quicly_stats); + + s = format (s, "[RTT >%3d, ~%3d, V%3d, last %3d]", + quicly_stats.rtt.minimum, quicly_stats.rtt.smoothed, + quicly_stats.rtt.variance, quicly_stats.rtt.latest); + s = format (s, " TX:%d RX:%d loss:%d ack:%d", + quicly_stats.num_packets.sent, + quicly_stats.num_packets.received, + quicly_stats.num_packets.lost, + quicly_stats.num_packets.ack_received); + return s; +} + +static u8 * +quic_format_stream_ctx (u8 * s, va_list * args) +{ + quic_ctx_t *ctx = va_arg (*args, quic_ctx_t *); + session_t *stream_session; + quicly_stream_t *stream = ctx->stream; + u32 txs, rxs; + + s = format (s, "[#%d][%x]", ctx->c_thread_index, ctx->c_c_index); + s = format (s, "[%U]", quic_format_quicly_stream_id, stream); + + stream_session = session_get_if_valid (ctx->c_s_index, ctx->c_thread_index); + if (!stream_session) + { + s = format (s, "- no session -\n"); + return s; + } + txs = svm_fifo_max_dequeue (stream_session->tx_fifo); + rxs = svm_fifo_max_dequeue (stream_session->rx_fifo); + s = format (s, "[rx %d tx %d]\n", rxs, txs); + return s; } static clib_error_t * -quic_show_ctx_command_fn (vlib_main_t * vm, unformat_input_t * input, - vlib_cli_command_t * cmd) +quic_show_connections_command_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) { + unformat_input_t _line_input, *line_input = &_line_input; + u8 show_listeners = 0, show_conn = 0, show_stream = 0; + u32 num_workers = vlib_num_workers (); quic_main_t *qm = &quic_main; + clib_error_t *error = 0; quic_ctx_t *ctx = NULL; - u32 num_workers = vlib_num_workers (); + + session_cli_return_if_not_enabled (); + + if (!unformat_user (input, unformat_line_input, line_input)) + { + quic_show_aggregated_stats (vm); + return 0; + } + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "listener")) + show_listeners = 1; + else if (unformat (line_input, "conn")) + show_conn = 1; + else if (unformat (line_input, "stream")) + show_stream = 1; + else + { + error = clib_error_return (0, "unknown input `%U'", + format_unformat_error, line_input); + goto done; + } + } for (int i = 0; i < num_workers + 1; i++) { /* *INDENT-OFF* */ pool_foreach (ctx, qm->ctx_pool[i], ({ - vlib_cli_output (vm, "%U", format_quic_ctx, ctx, 1); + if (quic_ctx_is_stream (ctx) && show_stream) + vlib_cli_output (vm, "%U", quic_format_stream_ctx, ctx); + else if (quic_ctx_is_listener (ctx) && show_listeners) + vlib_cli_output (vm, "%U", quic_format_listener_ctx, ctx); + else if (quic_ctx_is_conn (ctx) && show_conn) + vlib_cli_output (vm, "%U", quic_format_connection_ctx, ctx); })); /* *INDENT-ON* */ } - return 0; + +done: + unformat_free (line_input); + return error; } /* *INDENT-OFF* */ @@ -2341,17 +2824,23 @@ VLIB_CLI_COMMAND(quic_plugin_set_fifo_size_command, static)= .short_help = "quic set fifo-size N[K|M|G] (default 64K)", .function = quic_plugin_set_fifo_size_command_fn, }; -VLIB_CLI_COMMAND(quic_plugin_stats_command, static)= +VLIB_CLI_COMMAND(quic_show_ctx_command, static)= { - .path = "show quic stats", - .short_help = "show quic stats", - .function = quic_plugin_showstats_command_fn, + .path = "show quic", + .short_help = "show quic", + .function = quic_show_connections_command_fn, }; -VLIB_CLI_COMMAND(quic_show_ctx_command, static)= +VLIB_CLI_COMMAND (quic_list_crypto_context_command, static) = +{ + .path = "show quic crypto context", + .short_help = "list quic crypto contextes", + .function = quic_list_crypto_context_command_fn, +}; +VLIB_CLI_COMMAND (quic_set_max_packets_per_key, static) = { - .path = "show quic ctx", - .short_help = "show quic ctx", - .function = quic_show_ctx_command_fn, + .path = "set quic max_packets_per_key", + .short_help = "set quic max_packets_per_key 16777216", + .function = quic_set_max_packets_per_key_fn, }; VLIB_PLUGIN_REGISTER () = {