X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fplugins%2Fquic%2Fquic.c;h=14002a253d79f2a0c0b9b2105c1cab5eaa7724a0;hb=refs%2Fchanges%2F28%2F26328%2F2;hp=738290177cc2333e23c0d47e3110687a4505f307;hpb=d1b9e706841150d0da042a05aab51f19fa804192;p=vpp.git diff --git a/src/plugins/quic/quic.c b/src/plugins/quic/quic.c index 738290177cc..14002a253d7 100644 --- a/src/plugins/quic/quic.c +++ b/src/plugins/quic/quic.c @@ -26,10 +26,14 @@ #include #include #include -#include #include #include +#include + +#include + +extern quicly_crypto_engine_t quic_crypto_engine; static char *quic_error_strings[] = { #define quic_error(n,s) s, @@ -37,7 +41,9 @@ static char *quic_error_strings[] = { #undef quic_error }; -static quic_main_t quic_main; +#define DEFAULT_MAX_PACKETS_PER_KEY 16777216 + +quic_main_t quic_main; static void quic_update_timer (quic_ctx_t * ctx); static void quic_check_quic_session_connected (quic_ctx_t * ctx); static int quic_reset_connection (u64 udp_session_handle, @@ -85,6 +91,33 @@ quic_crypto_context_free_if_needed (crypto_context_t * crctx, u8 thread_index) pool_put (qm->wrk_ctx[thread_index].crypto_ctx_pool, crctx); } +static quicly_datagram_t * +quic_alloc_packet (quicly_packet_allocator_t * self, size_t payloadsize) +{ + quicly_datagram_t *packet; + if ((packet = + clib_mem_alloc (sizeof (*packet) + payloadsize + + sizeof (quic_encrypt_cb_ctx))) == NULL) + return NULL; + packet->data.base = + (uint8_t *) packet + sizeof (*packet) + sizeof (quic_encrypt_cb_ctx); + quic_encrypt_cb_ctx *encrypt_cb_ctx = + (quic_encrypt_cb_ctx *) ((uint8_t *) packet + sizeof (*packet)); + + clib_memset (encrypt_cb_ctx, 0, sizeof (*encrypt_cb_ctx)); + return packet; +} + +static void +quic_free_packet (quicly_packet_allocator_t * self, + quicly_datagram_t * packet) +{ + clib_mem_free (packet); +} + +quicly_packet_allocator_t quic_packet_allocator = + { quic_alloc_packet, quic_free_packet }; + static int quic_app_cert_key_pair_delete_callback (app_cert_key_pair_t * ckpair) { @@ -154,6 +187,32 @@ quic_list_crypto_context_command_fn (vlib_main_t * vm, return 0; } +static clib_error_t * +quic_set_max_packets_per_key_fn (vlib_main_t * vm, + unformat_input_t * input, + vlib_cli_command_t * cmd) +{ + quic_main_t *qm = &quic_main; + unformat_input_t _line_input, *line_input = &_line_input; + u64 tmp; + + if (!unformat_user (input, unformat_line_input, line_input)) + return 0; + + while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT) + { + if (unformat (line_input, "%U", unformat_memory_size, &tmp)) + { + qm->max_packets_per_key = tmp; + } + else + return clib_error_return (0, "unknown input '%U'", + format_unformat_error, line_input); + } + + return 0; +} + static void quic_release_crypto_context (u32 crypto_context_index, u8 thread_index) { @@ -203,16 +262,19 @@ quic_init_crypto_context (crypto_context_t * crctx, quic_ctx_t * ctx) clib_memcpy (quicly_ctx, &quicly_spec_context, sizeof (quicly_context_t)); quicly_ctx->max_packet_size = QUIC_MAX_PACKET_SIZE; + quicly_ctx->max_packets_per_key = qm->max_packets_per_key; quicly_ctx->tls = ptls_ctx; quicly_ctx->stream_open = &on_stream_open; quicly_ctx->closed_by_peer = &on_closed_by_peer; quicly_ctx->now = &quicly_vpp_now_cb; quicly_amend_ptls_context (quicly_ctx->tls); + quicly_ctx->packet_allocator = &quic_packet_allocator; + quicly_ctx->crypto_engine = &quic_crypto_engine; quicly_ctx->transport_params.max_data = QUIC_INT_MAX; quicly_ctx->transport_params.max_streams_uni = (uint64_t) 1 << 60; quicly_ctx->transport_params.max_streams_bidi = (uint64_t) 1 << 60; - quicly_ctx->transport_params.idle_timeout = qm->connection_timeout; + quicly_ctx->transport_params.max_idle_timeout = qm->connection_timeout; app = application_get (ctx->parent_app_id); quicly_ctx->transport_params.max_stream_data.bidi_local = @@ -350,7 +412,7 @@ quic_ctx_get_if_valid (u32 ctx_index, u32 thread_index) return pool_elt_at_index (quic_main.ctx_pool[thread_index], ctx_index); } -static quic_ctx_t * +quic_ctx_t * quic_get_conn_ctx (quicly_conn_t * conn) { u64 conn_data; @@ -633,12 +695,14 @@ quic_send_datagram (session_t * udp_session, quicly_datagram_t * packet) static int quic_send_packets (quic_ctx_t * ctx) { + quic_main_t *qm = &quic_main; quicly_datagram_t *packets[QUIC_SEND_PACKET_VEC_SIZE]; session_t *udp_session; quicly_conn_t *conn; size_t num_packets, i, max_packets; quicly_packet_allocator_t *pa; int err = 0; + u32 thread_index = vlib_get_thread_index (); /* We have sctx, get qctx */ if (quic_ctx_is_stream (ctx)) @@ -669,8 +733,12 @@ quic_send_packets (quic_ctx_t * ctx) if ((err = quicly_send (conn, packets, &num_packets))) goto quicly_error; + quic_crypto_batch_tx_packets (&qm->wrk_ctx + [thread_index].crypto_context_batch); + for (i = 0; i != num_packets; ++i) { + quic_crypto_finalize_send_packet (packets[i]); if ((err = quic_send_datagram (udp_session, packets[i]))) goto quicly_error; @@ -716,7 +784,7 @@ quic_on_stream_destroy (quicly_stream_t * stream, int err) clib_mem_free (stream->data); } -static int +static void quic_on_stop_sending (quicly_stream_t * stream, int err) { #if QUIC_DEBUG >= 2 @@ -729,10 +797,9 @@ quic_on_stop_sending (quicly_stream_t * stream, int err) session_handle (stream_session), quic_format_err, err); #endif /* TODO : handle STOP_SENDING */ - return 0; } -static int +static void quic_on_receive_reset (quicly_stream_t * stream, int err) { quic_stream_data_t *stream_data = (quic_stream_data_t *) stream->data; @@ -745,10 +812,9 @@ quic_on_receive_reset (quicly_stream_t * stream, int err) session_handle (stream_session), quic_format_err, err); #endif session_transport_closing_notify (&sctx->connection); - return 0; } -static int +static void quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, size_t len) { @@ -778,7 +844,7 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, stream_session->thread_index, f, max_enq, len, stream_data->app_rx_data_len, off, off - stream_data->app_rx_data_len + len); - return 0; + return; } if (PREDICT_FALSE ((off - stream_data->app_rx_data_len + len) > max_enq)) { @@ -790,7 +856,7 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, stream_session->thread_index, f, max_enq, len, stream_data->app_rx_data_len, off, off - stream_data->app_rx_data_len + len); - return 1; + return; /* This shouldn't happen */ } if (off == stream_data->app_rx_data_len) { @@ -820,7 +886,7 @@ quic_on_receive (quicly_stream_t * stream, size_t off, const void *src, len, (u8 *) src); QUIC_ASSERT (rlen == 0); } - return 0; + return; } void @@ -844,14 +910,14 @@ quic_fifo_egress_shift (quicly_stream_t * stream, size_t delta) QUIC_ASSERT (!rv); } -int +void quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst, size_t * len, int *wrote_all) { - u32 deq_max, first_deq, max_rd_chunk, rem_offset; quic_stream_data_t *stream_data; session_t *stream_session; svm_fifo_t *f; + u32 deq_max; stream_data = (quic_stream_data_t *) stream->data; stream_session = get_stream_session_from_stream (stream); @@ -875,24 +941,7 @@ quic_fifo_egress_emit (quicly_stream_t * stream, size_t off, void *dst, if (off + *len > stream_data->app_tx_data_len) stream_data->app_tx_data_len = off + *len; - /* TODO, use something like : return svm_fifo_peek (f, off, *len, dst); */ - max_rd_chunk = svm_fifo_max_read_chunk (f); - - first_deq = 0; - if (off < max_rd_chunk) - { - first_deq = clib_min (*len, max_rd_chunk - off); - clib_memcpy_fast (dst, svm_fifo_head (f) + off, first_deq); - } - - if (max_rd_chunk < off + *len) - { - rem_offset = max_rd_chunk < off ? off - max_rd_chunk : 0; - clib_memcpy_fast (dst + first_deq, f->head_chunk->data + rem_offset, - *len - first_deq); - } - - return 0; + svm_fifo_peek (f, off, *len, dst); } static const quicly_stream_callbacks_t quic_stream_callbacks = { @@ -1122,7 +1171,6 @@ quic_expired_timers_dispatch (u32 * expired_timers) } /* Transport proto functions */ - static int quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) { @@ -1134,6 +1182,7 @@ quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) app_worker_t *app_wrk; quic_ctx_t *qctx, *sctx; u32 sctx_index; + u8 is_unidir; int rv; /* Find base session to which the user want to attach a stream */ @@ -1177,9 +1226,8 @@ quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) if (!conn || !quicly_connection_is_ready (conn)) return -1; - if ((rv = - quicly_open_stream (conn, &stream, - sep->flags & SESSION_F_UNIDIRECTIONAL))) + is_unidir = sep->transport_flags & TRANSPORT_CFG_F_UNIDIRECTIONAL; + if ((rv = quicly_open_stream (conn, &stream, is_unidir))) { QUIC_DBG (2, "Stream open failed with %d", rv); return -1; @@ -1198,7 +1246,7 @@ quic_connect_stream (session_t * quic_session, session_endpoint_cfg_t * sep) stream_session->listener_handle = quic_session_handle; stream_session->session_type = session_type_from_proto_and_ip (TRANSPORT_PROTO_QUIC, qctx->udp_is_ip4); - if (sep->flags & SESSION_F_UNIDIRECTIONAL) + if (is_unidir) stream_session->flags |= SESSION_F_UNIDIRECTIONAL; sctx->c_s_index = stream_session->session_index; @@ -1264,7 +1312,7 @@ quic_connect_connection (session_endpoint_cfg_t * sep) vec_terminate_c_string (ctx->srv_hostname); clib_memcpy (&cargs->sep, sep, sizeof (session_endpoint_cfg_t)); - cargs->sep.transport_proto = TRANSPORT_PROTO_UDPC; + cargs->sep.transport_proto = TRANSPORT_PROTO_UDP; cargs->app_index = qm->app_index; cargs->api_context = ctx_index; @@ -1272,6 +1320,7 @@ quic_connect_connection (session_endpoint_cfg_t * sep) app = application_get (app_wrk->app_index); ctx->parent_app_id = app_wrk->app_index; cargs->sep_ext.ns_index = app->ns_index; + cargs->sep_ext.transport_flags = TRANSPORT_CFG_F_CONNECTED; ctx->crypto_engine = sep->crypto_engine; ctx->ckpair_index = sep->ckpair_index; @@ -1375,11 +1424,12 @@ quic_start_listen (u32 quic_listen_session_index, transport_endpoint_t * tep) app = application_get (app_wrk->app_index); QUIC_DBG (2, "Called quic_start_listen for app %d", app_wrk->app_index); - sep->transport_proto = TRANSPORT_PROTO_UDPC; clib_memset (args, 0, sizeof (*args)); args->app_index = qm->app_index; args->sep_ext = *sep; args->sep_ext.ns_index = app->ns_index; + args->sep_ext.transport_proto = TRANSPORT_PROTO_UDP; + args->sep_ext.transport_flags = TRANSPORT_CFG_F_CONNECTED; if ((rv = vnet_listen (args))) return rv; @@ -1607,6 +1657,19 @@ quic_check_quic_session_connected (quic_ctx_t * ctx) quic_on_quic_session_connected (ctx); } +static inline void +quic_update_conn_ctx (quicly_conn_t * conn, quicly_context_t * quicly_context) +{ + /* we need to update the quicly_conn on migrate + * as it contains a pointer to the crypto context */ + ptls_context_t **tls; + quicly_context_t **_quicly_context; + _quicly_context = (quicly_context_t **) conn; + *_quicly_context = quicly_context; + tls = (ptls_context_t **) quicly_get_tls (conn); + *tls = quicly_context->tls; +} + static void quic_receive_connection (void *arg) { @@ -1614,6 +1677,7 @@ quic_receive_connection (void *arg) quic_ctx_t *temp_ctx, *new_ctx; clib_bihash_kv_16_8_t kv; quicly_conn_t *conn; + quicly_context_t *quicly_context; session_t *udp_session; temp_ctx = arg; @@ -1631,6 +1695,9 @@ quic_receive_connection (void *arg) quic_acquire_crypto_context (new_ctx); conn = new_ctx->conn; + quicly_context = quic_get_quicly_ctx_from_ctx (new_ctx); + quic_update_conn_ctx (conn, quicly_context); + quic_store_conn_ctx (conn, new_ctx); quic_make_connection_key (&kv, quicly_get_master_id (conn)); kv.value = ((u64) thread_index) << 32 | (u64) new_ctx_id; @@ -1956,10 +2023,9 @@ quic_find_packet_ctx (quic_rx_packet_ctx_t * pctx, u32 caller_thread_index) return QUIC_PACKET_TYPE_RECEIVE; } -static int -quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) +static void +quic_accept_connection (quic_rx_packet_ctx_t * pctx) { - u32 thread_index = vlib_get_thread_index (); quicly_context_t *quicly_ctx; session_t *quic_session; clib_bihash_kv_16_8_t kv; @@ -1971,27 +2037,28 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) /* new connection, accept and create context if packet is valid * TODO: check if socket is actually listening? */ - ctx = quic_ctx_get (ctx_index, thread_index); + ctx = quic_ctx_get (pctx->ctx_index, pctx->thread_index); if (ctx->c_s_index != QUIC_SESSION_INVALID) { - QUIC_DBG (2, "already accepted ctx 0x%x", ctx_index); - return 0; + QUIC_DBG (2, "already accepted ctx 0x%x", ctx->c_s_index); + return; } quicly_ctx = quic_get_quicly_ctx_from_ctx (ctx); if ((rv = quicly_accept (&conn, quicly_ctx, NULL, &pctx->sa, &pctx->packet, NULL, - &quic_main.wrk_ctx[thread_index].next_cid, NULL))) + &quic_main.wrk_ctx[pctx->thread_index].next_cid, + NULL))) { /* Invalid packet, pass */ assert (conn == NULL); QUIC_ERR ("Accept failed with %U", quic_format_err, rv); /* TODO: cleanup created quic ctx and UDP session */ - return 0; + return; } assert (conn != NULL); - ++quic_main.wrk_ctx[thread_index].next_cid.master_id; + ++quic_main.wrk_ctx[pctx->thread_index].next_cid.master_id; /* Save ctx handle in quicly connection */ quic_store_conn_ctx (conn, ctx); ctx->conn = conn; @@ -2012,7 +2079,7 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) /* Register connection in connections map */ quic_make_connection_key (&kv, quicly_get_master_id (conn)); - kv.value = ((u64) thread_index) << 32 | (u64) ctx_index; + kv.value = ((u64) pctx->thread_index) << 32 | (u64) pctx->ctx_index; clib_bihash_add_del_16_8 (&quic_main.connection_hash, &kv, 1 /* is_add */ ); QUIC_DBG (2, "Registering conn with id %lu %lu", kv.key[0], kv.key[1]); @@ -2020,23 +2087,19 @@ quic_accept_connection (u32 ctx_index, quic_rx_packet_ctx_t * pctx) if ((rv = app_worker_init_accepted (quic_session))) { QUIC_ERR ("failed to allocate fifos"); - quic_proto_on_close (ctx_index, thread_index); - return rv; + quic_proto_on_close (pctx->ctx_index, pctx->thread_index); + return; } app_wrk = app_worker_get (quic_session->app_wrk_index); if ((rv = app_worker_accept_notify (app_wrk, quic_session))) { QUIC_ERR ("failed to notify accept worker app"); - quic_proto_on_close (ctx_index, thread_index); - return rv; + quic_proto_on_close (pctx->ctx_index, pctx->thread_index); + return; } ctx->conn_state = QUIC_CONN_STATE_READY; - pctx->ctx_index = ctx_index; - pctx->thread_index = thread_index; - - return 0; } static int @@ -2075,6 +2138,7 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, u32 thread_index = vlib_get_thread_index (); u32 cur_deq = svm_fifo_max_dequeue (f) - fifo_offset; quicly_context_t *quicly_ctx; + session_t *udp_session; int rv; ret = svm_fifo_peek (f, fifo_offset, @@ -2114,6 +2178,11 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, if (rv == QUIC_PACKET_TYPE_RECEIVE) { pctx->ptype = QUIC_PACKET_TYPE_RECEIVE; + if (quic_main.vnet_crypto_enabled) + { + quic_ctx_t *qctx = quic_ctx_get (pctx->ctx_index, thread_index); + quic_crypto_decrypt_packet (qctx, pctx); + } return 0; } else if (rv == QUIC_PACKET_TYPE_MIGRATE) @@ -2124,6 +2193,9 @@ quic_process_one_rx_packet (u64 udp_session_handle, svm_fifo_t * f, else if (QUICLY_PACKET_IS_LONG_HEADER (pctx->packet.octets.base[0])) { pctx->ptype = QUIC_PACKET_TYPE_ACCEPT; + udp_session = session_get_from_handle (udp_session_handle); + pctx->ctx_index = udp_session->opaque; + pctx->thread_index = thread_index; } else { @@ -2136,7 +2208,8 @@ static int quic_udp_session_rx_callback (session_t * udp_session) { /* Read data from UDP rx_fifo and pass it to the quicly conn. */ - quic_ctx_t *ctx = NULL; + quic_main_t *qm = &quic_main; + quic_ctx_t *ctx = NULL, *prev_ctx = NULL; svm_fifo_t *f = udp_session->rx_fifo; u32 max_deq; u64 udp_session_handle = session_handle (udp_session); @@ -2160,6 +2233,11 @@ rx_start: fifo_offset = 0; max_packets = QUIC_RCV_MAX_BATCH_PACKETS; +#if CLIB_DEBUG > 0 + clib_memset (packets_ctx, 0xfa, + QUIC_RCV_MAX_BATCH_PACKETS * sizeof (quic_rx_packet_ctx_t)); +#endif + for (i = 0; i < max_packets; i++) { packets_ctx[i].thread_index = UINT32_MAX; @@ -2190,6 +2268,9 @@ rx_start: } } + quic_crypto_batch_rx_packets (&qm-> + wrk_ctx[thread_index].crypto_context_batch); + for (i = 0; i < max_packets; i++) { switch (packets_ctx[i].ptype) @@ -2205,20 +2286,17 @@ rx_start: } break; case QUIC_PACKET_TYPE_ACCEPT: - udp_session = session_get_from_handle (udp_session_handle); - if ((rv = quic_accept_connection (udp_session->opaque, - &packets_ctx[i]))) - { - QUIC_ERR ("quic accept errored with %d", rv); - } + quic_accept_connection (&packets_ctx[i]); break; case QUIC_PACKET_TYPE_RESET: quic_reset_connection (udp_session_handle, &packets_ctx[i]); break; } } + ctx = prev_ctx = NULL; for (i = 0; i < max_packets; i++) { + prev_ctx = ctx; switch (packets_ctx[i].ptype) { case QUIC_PACKET_TYPE_RECEIVE: @@ -2233,9 +2311,11 @@ rx_start: packets_ctx[i].thread_index); break; default: - continue; + continue; /* this exits the for loop since other packet types are + necessarily the last in the batch */ } - quic_send_packets (ctx); + if (ctx != prev_ctx) + quic_send_packets (ctx); } udp_session = session_get_from_handle (udp_session_handle); /* session alloc might have happened */ @@ -2316,6 +2396,8 @@ static const transport_proto_vft_t quic_proto = { .get_transport_endpoint = quic_get_transport_endpoint, .get_transport_listener_endpoint = quic_get_transport_listener_endpoint, .transport_options = { + .name = "quic", + .short_name = "Q", .tx_type = TRANSPORT_TX_INTERNAL, .service_type = TRANSPORT_SERVICE_APP, }, @@ -2391,6 +2473,7 @@ quic_init (vlib_main_t * vm) vec_validate (qm->ctx_pool, num_threads - 1); vec_validate (qm->wrk_ctx, num_threads - 1); + for (i = 0; i < num_threads; i++) { qm->wrk_ctx[i].next_cid.thread_id = i; @@ -2400,6 +2483,9 @@ quic_init (vlib_main_t * vm) tw->last_run_time = vlib_time_now (vlib_get_main ()); clib_bihash_init_24_8 (&qm->wrk_ctx[i].crypto_context_hash, "quic crypto contexts", 64, 128 << 10); + + qm->wrk_ctx[i].crypto_context_batch.nb_rx_packets = 0; + qm->wrk_ctx[i].crypto_context_batch.nb_tx_packets = 0; } clib_bihash_init_16_8 (&qm->connection_hash, "quic connections", 1024, @@ -2420,7 +2506,16 @@ quic_init (vlib_main_t * vm) quic_register_cipher_suite (CRYPTO_ENGINE_VPP, quic_crypto_cipher_suites); quic_register_cipher_suite (CRYPTO_ENGINE_PICOTLS, ptls_openssl_cipher_suites); - qm->default_crypto_engine = CRYPTO_ENGINE_PICOTLS; + qm->default_crypto_engine = CRYPTO_ENGINE_VPP; + qm->max_packets_per_key = DEFAULT_MAX_PACKETS_PER_KEY; + clib_rwlock_init (&qm->crypto_keys_quic_rw_lock); + + vnet_crypto_main_t *cm = &crypto_main; + if (vec_len (cm->engines) == 0) + qm->vnet_crypto_enabled = 0; + else + qm->vnet_crypto_enabled = 1; + vec_free (a->name); return 0; } @@ -2739,6 +2834,12 @@ VLIB_CLI_COMMAND (quic_list_crypto_context_command, static) = .short_help = "list quic crypto contextes", .function = quic_list_crypto_context_command_fn, }; +VLIB_CLI_COMMAND (quic_set_max_packets_per_key, static) = +{ + .path = "set quic max_packets_per_key", + .short_help = "set quic max_packets_per_key 16777216", + .function = quic_set_max_packets_per_key_fn, +}; VLIB_PLUGIN_REGISTER () = { .version = VPP_BUILD_VER,