X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp.c;h=5a215b658a0e8b44cc587637a39d1e8c0c4d4241;hb=26dd6de91b4d36ac04154c7eb6339684db6684a0;hp=cb05b8c053327854b5341b1c769f641906a0fc8d;hpb=bf4d5ce58435d3f424749ff69650ea67ce778f04;p=vpp.git diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index cb05b8c0533..5a215b658a0 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Cisco and/or its affiliates. + * Copyright (c) 2016-2019 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -25,6 +25,7 @@ #include #include #include +#include tcp_main_t tcp_main; @@ -69,6 +70,46 @@ tcp_add_del_adjacency (tcp_connection_t * tc, u8 is_add) sizeof (args)); } +static void +tcp_cc_init (tcp_connection_t * tc) +{ + tc->cc_algo = tcp_cc_algo_get (tcp_main.cc_algo); + tc->cc_algo->init (tc); +} + +static void +tcp_cc_cleanup (tcp_connection_t * tc) +{ + if (tc->cc_algo->cleanup) + tc->cc_algo->cleanup (tc); +} + +void +tcp_cc_algo_register (tcp_cc_algorithm_type_e type, + const tcp_cc_algorithm_t * vft) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + vec_validate (tm->cc_algos, type); + + tm->cc_algos[type] = *vft; + hash_set_mem (tm->cc_algo_by_name, vft->name, type); +} + +tcp_cc_algorithm_t * +tcp_cc_algo_get (tcp_cc_algorithm_type_e type) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + return &tm->cc_algos[type]; +} + +tcp_cc_algorithm_type_e +tcp_cc_algo_new_type (const tcp_cc_algorithm_t * vft) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + tcp_cc_algo_register (++tm->cc_last_type, vft); + return tm->cc_last_type; +} + static u32 tcp_connection_bind (u32 session_index, transport_endpoint_t * lcl) { @@ -172,7 +213,7 @@ tcp_half_open_connection_cleanup (tcp_connection_t * tc) /* Make sure this is the owning thread */ if (tc->c_thread_index != vlib_get_thread_index ()) return 1; - tcp_timer_reset (tc, TCP_TIMER_ESTABLISH); + tcp_timer_reset (tc, TCP_TIMER_ESTABLISH_AO); tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT_SYN); tcp_half_open_connection_del (tc); return 0; @@ -200,6 +241,8 @@ tcp_connection_cleanup (tcp_connection_t * tc) { tcp_main_t *tm = &tcp_main; + TCP_EVT_DBG (TCP_EVT_DELETE, tc); + /* Cleanup local endpoint if this was an active connect */ transport_endpoint_cleanup (TRANSPORT_PROTO_TCP, &tc->c_lcl_ip, tc->c_lcl_port); @@ -223,6 +266,13 @@ tcp_connection_cleanup (tcp_connection_t * tc) if (!tc->c_is_ip4 && ip6_address_is_link_local_unicast (&tc->c_rmt_ip6)) tcp_add_del_adjacency (tc, 0); + tcp_cc_cleanup (tc); + vec_free (tc->snd_sacks); + vec_free (tc->snd_sacks_fl); + + if (tc->flags & TCP_CONN_RATE_SAMPLE) + tcp_bt_cleanup (tc); + /* Poison the entry */ if (CLIB_DEBUG > 0) clib_memset (tc, 0xFA, sizeof (*tc)); @@ -240,13 +290,12 @@ tcp_connection_cleanup (tcp_connection_t * tc) void tcp_connection_del (tcp_connection_t * tc) { - TCP_EVT_DBG (TCP_EVT_DELETE, tc); - stream_session_delete_notify (&tc->connection); + session_transport_delete_notify (&tc->connection); tcp_connection_cleanup (tc); } tcp_connection_t * -tcp_connection_new (u8 thread_index) +tcp_connection_alloc (u8 thread_index) { tcp_main_t *tm = vnet_get_tcp_main (); tcp_connection_t *tc; @@ -258,6 +307,20 @@ tcp_connection_new (u8 thread_index) return tc; } +void +tcp_connection_free (tcp_connection_t * tc) +{ + tcp_main_t *tm = &tcp_main; + if (CLIB_DEBUG) + { + u8 thread_index = tc->c_thread_index; + clib_memset (tc, 0xFA, sizeof (*tc)); + pool_put (tm->connections[thread_index], tc); + return; + } + pool_put (tm->connections[tc->c_thread_index], tc); +} + /** Notify session that connection has been reset. * * Switch state to closed and wait for session to call cleanup. @@ -270,7 +333,7 @@ tcp_connection_reset (tcp_connection_t * tc) { case TCP_STATE_SYN_RCVD: /* Cleanup everything. App wasn't notified yet */ - stream_session_delete_notify (&tc->connection); + session_transport_delete_notify (&tc->connection); tcp_connection_cleanup (tc); break; case TCP_STATE_SYN_SENT: @@ -281,20 +344,28 @@ tcp_connection_reset (tcp_connection_t * tc) tcp_connection_timers_reset (tc); /* Set the cleanup timer, in case the session layer/app don't * cleanly close the connection */ - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); - stream_session_reset_notify (&tc->connection); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME); + session_transport_reset_notify (&tc->connection); + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + session_transport_closed_notify (&tc->connection); break; case TCP_STATE_CLOSE_WAIT: case TCP_STATE_FIN_WAIT_1: case TCP_STATE_FIN_WAIT_2: case TCP_STATE_CLOSING: - tc->state = TCP_STATE_CLOSED; - TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc); + case TCP_STATE_LAST_ACK: tcp_connection_timers_reset (tc); - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME); + /* Make sure we mark the session as closed. In some states we may + * be still trying to send data */ + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + session_transport_closed_notify (&tc->connection); break; case TCP_STATE_CLOSED: - return; + case TCP_STATE_TIME_WAIT: + break; + default: + TCP_DBG ("reset state: %u", tc->state); } } @@ -320,27 +391,43 @@ tcp_connection_close (tcp_connection_t * tc) switch (tc->state) { case TCP_STATE_SYN_SENT: - tc->state = TCP_STATE_CLOSED; + /* Try to cleanup. If not on the right thread, mark as half-open done. + * Connection will be cleaned up when establish timer pops */ + tcp_connection_cleanup (tc); break; case TCP_STATE_SYN_RCVD: tcp_connection_timers_reset (tc); tcp_send_fin (tc); - tc->state = TCP_STATE_FIN_WAIT_1; - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + tcp_connection_set_state (tc, TCP_STATE_FIN_WAIT_1); + tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_FINWAIT1_TIME); break; case TCP_STATE_ESTABLISHED: - if (!session_tx_fifo_max_dequeue (&tc->connection)) + /* If closing with unread data, reset the connection */ + if (transport_max_rx_dequeue (&tc->connection)) + { + tcp_send_reset (tc); + tcp_connection_timers_reset (tc); + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLOSEWAIT_TIME); + session_transport_closed_notify (&tc->connection); + break; + } + if (!transport_max_tx_dequeue (&tc->connection)) tcp_send_fin (tc); else tc->flags |= TCP_CONN_FINPNDG; - tc->state = TCP_STATE_FIN_WAIT_1; + tcp_connection_set_state (tc, TCP_STATE_FIN_WAIT_1); + /* Set a timer in case the peer stops responding. Otherwise the + * connection will be stuck here forever. */ + ASSERT (tc->timers[TCP_TIMER_WAITCLOSE] == TCP_TIMER_HANDLE_INVALID); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_FINWAIT1_TIME); break; case TCP_STATE_CLOSE_WAIT: - if (!session_tx_fifo_max_dequeue (&tc->connection)) + if (!transport_max_tx_dequeue (&tc->connection)) { tcp_send_fin (tc); tcp_connection_timers_reset (tc); - tc->state = TCP_STATE_LAST_ACK; + tcp_connection_set_state (tc, TCP_STATE_LAST_ACK); tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME); } else @@ -349,16 +436,16 @@ tcp_connection_close (tcp_connection_t * tc) case TCP_STATE_FIN_WAIT_1: tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME); break; + case TCP_STATE_CLOSED: + tcp_connection_timers_reset (tc); + /* Delete connection but instead of doing it now wait until next + * dispatch cycle to give the session layer a chance to clear + * unhandled events */ + tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + break; default: TCP_DBG ("state: %u", tc->state); } - - TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc); - - /* If in CLOSED and WAITCLOSE timer is not set, delete connection now */ - if (!tcp_timer_is_active (tc, TCP_TIMER_WAITCLOSE) - && tc->state == TCP_STATE_CLOSED) - tcp_connection_del (tc); } static void @@ -374,9 +461,7 @@ tcp_session_cleanup (u32 conn_index, u32 thread_index) { tcp_connection_t *tc; tc = tcp_connection_get (conn_index, thread_index); - tcp_connection_timers_reset (tc); - tc->state = TCP_STATE_CLOSED; - TCP_EVT_DBG (TCP_EVT_STATE_CHANGE, tc); + tcp_connection_set_state (tc, TCP_STATE_CLOSED); tcp_connection_cleanup (tc); } @@ -448,10 +533,10 @@ tcp_connection_select_lb_bucket (tcp_connection_t * tc, const dpo_id_t * dpo, ip6_tcp_hdr_t hdr; clib_memset (&hdr, 0, sizeof (hdr)); hdr.ip.protocol = IP_PROTOCOL_TCP; - clib_memcpy (&hdr.ip.src_address, &tc->c_lcl_ip.ip6, - sizeof (ip6_address_t)); - clib_memcpy (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6, - sizeof (ip6_address_t)); + clib_memcpy_fast (&hdr.ip.src_address, &tc->c_lcl_ip.ip6, + sizeof (ip6_address_t)); + clib_memcpy_fast (&hdr.ip.dst_address, &tc->c_rmt_ip.ip6, + sizeof (ip6_address_t)); hdr.tcp.src_port = tc->c_lcl_port; hdr.tcp.dst_port = tc->c_rmt_port; hash = ip6_compute_flow_hash (&hdr.ip, lb->lb_hash_config); @@ -466,7 +551,7 @@ tcp_lookup_rmt_in_fib (tcp_connection_t * tc) fib_prefix_t prefix; u32 fib_index; - clib_memcpy (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr)); + clib_memcpy_fast (&prefix.fp_addr, &tc->c_rmt_ip, sizeof (prefix.fp_addr)); prefix.fp_proto = tc->c_is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; prefix.fp_len = tc->c_is_ip4 ? 32 : 128; fib_index = fib_table_find (prefix.fp_proto, tc->c_fib_index); @@ -507,52 +592,57 @@ tcp_connection_fib_attach (tcp_connection_t * tc) } #endif /* 0 */ -static void -tcp_cc_init (tcp_connection_t * tc) -{ - tc->cc_algo = tcp_cc_algo_get (TCP_CC_NEWRENO); - tc->cc_algo->init (tc); -} - -void -tcp_cc_algo_register (tcp_cc_algorithm_type_e type, - const tcp_cc_algorithm_t * vft) +/** + * Generate random iss as per rfc6528 + */ +static u32 +tcp_generate_random_iss (tcp_connection_t * tc) { - tcp_main_t *tm = vnet_get_tcp_main (); - vec_validate (tm->cc_algos, type); + tcp_main_t *tm = &tcp_main; + u64 tmp; - tm->cc_algos[type] = *vft; -} + if (tc->c_is_ip4) + tmp = (u64) tc->c_lcl_ip.ip4.as_u32 << 32 | (u64) tc->c_rmt_ip.ip4.as_u32; + else + tmp = tc->c_lcl_ip.ip6.as_u64[0] ^ tc->c_lcl_ip.ip6.as_u64[1] + ^ tc->c_rmt_ip.ip6.as_u64[0] ^ tc->c_rmt_ip.ip6.as_u64[1]; -tcp_cc_algorithm_t * -tcp_cc_algo_get (tcp_cc_algorithm_type_e type) -{ - tcp_main_t *tm = vnet_get_tcp_main (); - return &tm->cc_algos[type]; + tmp ^= tm->iss_seed.first | ((u64) tc->c_lcl_port << 16 | tc->c_rmt_port); + tmp ^= tm->iss_seed.second; + tmp = clib_xxhash (tmp) + clib_cpu_time_now (); + return ((tmp >> 32) ^ (tmp & 0xffffffff)); } - /** * Initialize connection send variables. */ void tcp_init_snd_vars (tcp_connection_t * tc) { - u32 time_now; - /* * We use the time to randomize iss and for setting up the initial * timestamp. Make sure it's updated otherwise syn and ack in the * handshake may make it look as if time has flown in the opposite * direction for us. */ - tcp_set_time_now (vlib_get_thread_index ()); - time_now = tcp_time_now (); + tcp_set_time_now (tcp_get_worker (vlib_get_thread_index ())); - tc->iss = random_u32 (&time_now); + tc->iss = tcp_generate_random_iss (tc); tc->snd_una = tc->iss; tc->snd_nxt = tc->iss + 1; tc->snd_una_max = tc->snd_nxt; + tc->srtt = 0; +} + +void +tcp_enable_pacing (tcp_connection_t * tc) +{ + u32 initial_bucket, byte_rate; + initial_bucket = 16 * tc->snd_mss; + byte_rate = 2 << 16; + transport_connection_tx_pacer_init (&tc->connection, byte_rate, + initial_bucket); + tc->mrtt_us = (u32) ~ 0; } /** Initialize tcp connection variables @@ -572,7 +662,14 @@ tcp_connection_init_vars (tcp_connection_t * tc) if (!tc->c_is_ip4 && ip6_address_is_link_local_unicast (&tc->c_rmt_ip6)) tcp_add_del_adjacency (tc, 1); - // tcp_connection_fib_attach (tc); + /* tcp_connection_fib_attach (tc); */ + + if (transport_connection_is_tx_paced (&tc->connection) + || tcp_main.tx_pacing) + tcp_enable_pacing (tc); + + if (tc->flags & TCP_CONN_RATE_SAMPLE) + tcp_bt_init (tc); } static int @@ -592,8 +689,8 @@ tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr, index = tm->last_v6_address_rotor++; if (tm->last_v6_address_rotor >= vec_len (tm->ip6_src_addresses)) tm->last_v6_address_rotor = 0; - clib_memcpy (&lcl_addr->ip6, &tm->ip6_src_addresses[index], - sizeof (ip6_address_t)); + clib_memcpy_fast (&lcl_addr->ip6, &tm->ip6_src_addresses[index], + sizeof (ip6_address_t)); } port = transport_alloc_local_port (TRANSPORT_PROTO_TCP, lcl_addr); if (port < 1) @@ -606,7 +703,7 @@ tcp_alloc_custom_local_endpoint (tcp_main_t * tm, ip46_address_t * lcl_addr, } static int -tcp_connection_open (transport_endpoint_t * rmt) +tcp_session_open (transport_endpoint_cfg_t * rmt) { tcp_main_t *tm = vnet_get_tcp_main (); tcp_connection_t *tc; @@ -652,12 +749,6 @@ tcp_connection_open (transport_endpoint_t * rmt) return tc->c_c_index; } -static int -tcp_session_open (transport_endpoint_t * tep) -{ - return tcp_connection_open (tep); -} - const char *tcp_dbg_evt_str[] = { #define _(sym, str) str, foreach_tcp_dbg_evt @@ -723,7 +814,6 @@ format_tcp_timers (u8 * s, va_list * args) if (tc->timers[i] != TCP_TIMER_HANDLE_INVALID) last = i; - s = format (s, "["); for (i = 0; i < last; i++) { if (tc->timers[i] != TCP_TIMER_HANDLE_INVALID) @@ -731,9 +821,7 @@ format_tcp_timers (u8 * s, va_list * args) } if (last >= 0) - s = format (s, "%s]", tcp_conn_timers[i]); - else - s = format (s, "]"); + s = format (s, "%s", tcp_conn_timers[i]); return s; } @@ -757,12 +845,30 @@ tcp_rcv_wnd_available (tcp_connection_t * tc) return (i32) tc->rcv_wnd - (tc->rcv_nxt - tc->rcv_las); } +static u8 * +format_tcp_congestion (u8 * s, va_list * args) +{ + tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); + u32 indent = format_get_indent (s); + + s = format (s, "%U ", format_tcp_congestion_status, tc); + s = format (s, "algo %s cwnd %u ssthresh %u bytes_acked %u\n", + tc->cc_algo->name, tc->cwnd, tc->ssthresh, tc->bytes_acked); + s = format (s, "%Ucc space %u prev_cwnd %u prev_ssthresh %u rtx_bytes %u\n", + format_white_space, indent, tcp_available_cc_snd_space (tc), + tc->prev_cwnd, tc->prev_ssthresh, tc->snd_rxt_bytes); + s = format (s, "%Usnd_congestion %u dupack %u limited_transmit %u\n", + format_white_space, indent, tc->snd_congestion - tc->iss, + tc->rcv_dupacks, tc->limited_transmit - tc->iss); + return s; +} + static u8 * format_tcp_vars (u8 * s, va_list * args) { tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); - s = format (s, " flags: %U timers: %U\n", format_tcp_connection_flags, tc, - format_tcp_timers, tc); + s = format (s, " index: %u flags: %U timers: %U\n", tc->c_c_index, + format_tcp_connection_flags, tc, format_tcp_timers, tc); s = format (s, " snd_una %u snd_nxt %u snd_una_max %u", tc->snd_una - tc->iss, tc->snd_nxt - tc->iss, tc->snd_una_max - tc->iss); @@ -772,26 +878,25 @@ format_tcp_vars (u8 * s, va_list * args) tc->snd_wnd, tc->rcv_wnd, tc->rcv_wscale); s = format (s, "snd_wl1 %u snd_wl2 %u\n", tc->snd_wl1 - tc->irs, tc->snd_wl2 - tc->iss); - s = format (s, " flight size %u out space %u cc space %u rcv_wnd_av %u\n", + s = format (s, " flight size %u out space %u rcv_wnd_av %u", tcp_flight_size (tc), tcp_available_output_snd_space (tc), - tcp_available_cc_snd_space (tc), tcp_rcv_wnd_available (tc)); - s = format (s, " cong %U ", format_tcp_congestion_status, tc); - s = format (s, "cwnd %u ssthresh %u rtx_bytes %u bytes_acked %u\n", - tc->cwnd, tc->ssthresh, tc->snd_rxt_bytes, tc->bytes_acked); - s = format (s, " prev_ssthresh %u snd_congestion %u dupack %u", - tc->prev_ssthresh, tc->snd_congestion - tc->iss, - tc->rcv_dupacks); - s = format (s, " limited_transmit %u\n", tc->limited_transmit - tc->iss); - s = format (s, " tsecr %u tsecr_last_ack %u\n", tc->rcv_opts.tsecr, - tc->tsecr_last_ack); - s = format (s, " rto %u rto_boff %u srtt %u rttvar %u rtt_ts %u ", tc->rto, - tc->rto_boff, tc->srtt, tc->rttvar, tc->rtt_ts); - s = format (s, "rtt_seq %u\n", tc->rtt_seq); - s = format (s, " tsval_recent %u tsval_recent_age %u\n", tc->tsval_recent, + tcp_rcv_wnd_available (tc)); + s = format (s, " tsval_recent %u\n", tc->tsval_recent); + s = format (s, " tsecr %u tsecr_last_ack %u tsval_recent_age %u", + tc->rcv_opts.tsecr, tc->tsecr_last_ack, tcp_time_now () - tc->tsval_recent_age); + s = format (s, " snd_mss %u\n", tc->snd_mss); + s = format (s, " rto %u rto_boff %u srtt %u us %.3f rttvar %u rtt_ts %.4f", + tc->rto, tc->rto_boff, tc->srtt, tc->mrtt_us * 1000, tc->rttvar, + tc->rtt_ts); + s = format (s, " rtt_seq %u\n", tc->rtt_seq - tc->iss); + s = format (s, " cong: %U", format_tcp_congestion, tc); + if (tc->state >= TCP_STATE_ESTABLISHED) - s = format (s, " scoreboard: %U\n", format_tcp_scoreboard, &tc->sack_sb, - tc); + { + s = format (s, " sboard: %U\n", format_tcp_scoreboard, &tc->sack_sb, + tc); + } if (vec_len (tc->snd_sacks)) s = format (s, " sacks tx: %U\n", format_tcp_sacks, tc); @@ -806,15 +911,15 @@ format_tcp_connection_id (u8 * s, va_list * args) return s; if (tc->c_is_ip4) { - s = format (s, "[#%d][%s] %U:%d->%U:%d", tc->c_thread_index, "T", - format_ip4_address, &tc->c_lcl_ip4, + s = format (s, "[%d:%d][%s] %U:%d->%U:%d", tc->c_thread_index, + tc->c_s_index, "T", format_ip4_address, &tc->c_lcl_ip4, clib_net_to_host_u16 (tc->c_lcl_port), format_ip4_address, &tc->c_rmt_ip4, clib_net_to_host_u16 (tc->c_rmt_port)); } else { - s = format (s, "[#%d][%s] %U:%d->%U:%d", tc->c_thread_index, "T", - format_ip6_address, &tc->c_lcl_ip6, + s = format (s, "[%d:%d][%s] %U:%d->%U:%d", tc->c_thread_index, + tc->c_s_index, "T", format_ip6_address, &tc->c_lcl_ip6, clib_net_to_host_u16 (tc->c_lcl_port), format_ip6_address, &tc->c_rmt_ip6, clib_net_to_host_u16 (tc->c_rmt_port)); } @@ -861,14 +966,20 @@ static u8 * format_tcp_listener_session (u8 * s, va_list * args) { u32 tci = va_arg (*args, u32); + u32 __clib_unused thread_index = va_arg (*args, u32); + u32 verbose = va_arg (*args, u32); tcp_connection_t *tc = tcp_listener_get (tci); - return format (s, "%U", format_tcp_connection_id, tc); + s = format (s, "%-50U", format_tcp_connection_id, tc); + if (verbose) + s = format (s, "%-15U", format_tcp_state, tc->state); + return s; } static u8 * format_tcp_half_open_session (u8 * s, va_list * args) { u32 tci = va_arg (*args, u32); + u32 __clib_unused thread_index = va_arg (*args, u32); tcp_connection_t *tc = tcp_half_open_connection_get (tci); return format (s, "%U", format_tcp_connection_id, tc); } @@ -939,19 +1050,22 @@ format_tcp_scoreboard (u8 * s, va_list * args) sack_scoreboard_t *sb = va_arg (*args, sack_scoreboard_t *); tcp_connection_t *tc = va_arg (*args, tcp_connection_t *); sack_scoreboard_hole_t *hole; + u32 indent = format_get_indent (s); + s = format (s, "sacked_bytes %u last_sacked_bytes %u lost_bytes %u\n", sb->sacked_bytes, sb->last_sacked_bytes, sb->lost_bytes); - s = format (s, " last_bytes_delivered %u high_sacked %u snd_una_adv %u\n", - sb->last_bytes_delivered, sb->high_sacked - tc->iss, - sb->snd_una_adv); - s = format (s, " cur_rxt_hole %u high_rxt %u rescue_rxt %u", - sb->cur_rxt_hole, sb->high_rxt - tc->iss, - sb->rescue_rxt - tc->iss); + s = format (s, "%Ulast_bytes_delivered %u high_sacked %u snd_una_adv %u\n", + format_white_space, indent, sb->last_bytes_delivered, + sb->high_sacked - tc->iss, sb->snd_una_adv); + s = format (s, "%Ucur_rxt_hole %u high_rxt %u rescue_rxt %u", + format_white_space, indent, sb->cur_rxt_hole, + sb->high_rxt - tc->iss, sb->rescue_rxt - tc->iss); hole = scoreboard_first_hole (sb); if (hole) - s = format (s, "\n head %u tail %u %u holes:\n", sb->head, sb->tail, - pool_elts (sb->holes)); + s = format (s, "\n%Uhead %u tail %u %u holes:\n%U", format_white_space, + indent, sb->head, sb->tail, pool_elts (sb->holes), + format_white_space, indent); while (hole) { @@ -1028,38 +1142,26 @@ tcp_snd_space_inline (tcp_connection_t * tc) { int snd_space, snt_limited; - if (PREDICT_TRUE (!tcp_in_fastrecovery (tc))) - { - snd_space = tcp_available_output_snd_space (tc); + if (PREDICT_FALSE (tcp_in_fastrecovery (tc) + || tc->state == TCP_STATE_CLOSED)) + return 0; - /* If we haven't gotten dupacks or if we did and have gotten sacked - * bytes then we can still send as per Limited Transmit (RFC3042) */ - if (PREDICT_FALSE (tc->rcv_dupacks != 0 - && (tcp_opts_sack_permitted (tc) - && tc->sack_sb.last_sacked_bytes == 0))) - { - if (tc->rcv_dupacks == 1 && tc->limited_transmit != tc->snd_nxt) - tc->limited_transmit = tc->snd_nxt; - ASSERT (seq_leq (tc->limited_transmit, tc->snd_nxt)); - - snt_limited = tc->snd_nxt - tc->limited_transmit; - snd_space = clib_max (2 * tc->snd_mss - snt_limited, 0); - } - return tcp_round_snd_space (tc, snd_space); - } + snd_space = tcp_available_output_snd_space (tc); - /* RFC 5681: When previously unsent data is available and the new value of - * cwnd and the receiver's advertised window allow, a TCP SHOULD send 1*SMSS - * bytes of previously unsent data. */ - if (tcp_in_fastrecovery (tc) && !tcp_fastrecovery_sent_1_smss (tc)) + /* If we haven't gotten dupacks or if we did and have gotten sacked + * bytes then we can still send as per Limited Transmit (RFC3042) */ + if (PREDICT_FALSE (tc->rcv_dupacks != 0 + && (tcp_opts_sack_permitted (tc) + && tc->sack_sb.last_sacked_bytes == 0))) { - if (tcp_available_cc_snd_space (tc) < tc->snd_mss) - return 0; - tcp_fastrecovery_1_smss_on (tc); - return tc->snd_mss; - } + if (tc->rcv_dupacks == 1 && tc->limited_transmit != tc->snd_nxt) + tc->limited_transmit = tc->snd_nxt; + ASSERT (seq_leq (tc->limited_transmit, tc->snd_nxt)); - return 0; + snt_limited = tc->snd_nxt - tc->limited_transmit; + snd_space = clib_max (2 * tc->snd_mss - snt_limited, 0); + } + return tcp_round_snd_space (tc, snd_space); } u32 @@ -1090,45 +1192,78 @@ tcp_session_tx_fifo_offset (transport_connection_t * trans_conn) static void tcp_update_time (f64 now, u8 thread_index) { - tcp_set_time_now (thread_index); - tw_timer_expire_timers_16t_2w_512sl (&tcp_main. - wrk_ctx[thread_index].timer_wheel, - now); - tcp_do_fastretransmits (thread_index); - tcp_flush_frames_to_output (thread_index); + tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index); + + tcp_set_time_now (wrk); + tw_timer_expire_timers_16t_2w_512sl (&wrk->timer_wheel, now); + tcp_flush_frames_to_output (wrk); } -static u32 -tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) +static void +tcp_session_flush_data (transport_connection_t * tconn) { tcp_connection_t *tc = (tcp_connection_t *) tconn; - return tcp_push_header (tc, b); + if (tc->flags & TCP_CONN_PSH_PENDING) + return; + tc->flags |= TCP_CONN_PSH_PENDING; + tc->psh_seq = tc->snd_una + transport_max_tx_dequeue (tconn) - 1; } /* *INDENT-OFF* */ const static transport_proto_vft_t tcp_proto = { .enable = vnet_tcp_enable_disable, - .bind = tcp_session_bind, - .unbind = tcp_session_unbind, + .start_listen = tcp_session_bind, + .stop_listen = tcp_session_unbind, .push_header = tcp_session_push_header, .get_connection = tcp_session_get_transport, .get_listener = tcp_session_get_listener, .get_half_open = tcp_half_open_session_get_transport, - .open = tcp_session_open, + .connect = tcp_session_open, .close = tcp_session_close, .cleanup = tcp_session_cleanup, .send_mss = tcp_session_send_mss, .send_space = tcp_session_send_space, .update_time = tcp_update_time, .tx_fifo_offset = tcp_session_tx_fifo_offset, + .flush_data = tcp_session_flush_data, + .custom_tx = tcp_session_custom_tx, .format_connection = format_tcp_session, .format_listener = format_tcp_listener_session, .format_half_open = format_tcp_half_open_session, - .tx_type = TRANSPORT_TX_PEEK, - .service_type = TRANSPORT_SERVICE_VC, + .transport_options = { + .tx_type = TRANSPORT_TX_PEEK, + .service_type = TRANSPORT_SERVICE_VC, + }, }; /* *INDENT-ON* */ +void +tcp_connection_tx_pacer_update (tcp_connection_t * tc) +{ + f64 srtt; + u64 rate; + + if (!transport_connection_is_tx_paced (&tc->connection)) + return; + + srtt = clib_min ((f64) tc->srtt * TCP_TICK, tc->mrtt_us); + /* TODO should constrain to interface's max throughput but + * we don't have link speeds for sw ifs ..*/ + rate = tc->cwnd / srtt; + transport_connection_tx_pacer_update (&tc->connection, rate); +} + +void +tcp_connection_tx_pacer_reset (tcp_connection_t * tc, u32 window, + u32 start_bucket) +{ + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + u32 byte_rate = window / ((f64) TCP_TICK * tc->srtt); + u64 last_time = wrk->vm->clib_time.last_cpu_time; + transport_connection_tx_pacer_reset (&tc->connection, byte_rate, + start_bucket, last_time); +} + static void tcp_timer_keep_handler (u32 conn_index) { @@ -1146,26 +1281,35 @@ tcp_timer_establish_handler (u32 conn_index) { tcp_connection_t *tc; - tc = tcp_half_open_connection_get (conn_index); - if (tc) - { - ASSERT (tc->state == TCP_STATE_SYN_SENT); - session_stream_connect_notify (&tc->connection, 1 /* fail */ ); - TCP_DBG ("establish pop: %U", format_tcp_connection, tc, 2); - } - else - { - tc = tcp_connection_get (conn_index, vlib_get_thread_index ()); - /* note: the connection may have already disappeared */ - if (PREDICT_FALSE (tc == 0)) - return; - TCP_DBG ("establish pop: %U", format_tcp_connection, tc, 2); - ASSERT (tc->state == TCP_STATE_SYN_RCVD); - /* Start cleanup. App wasn't notified yet so use delete notify as - * opposed to delete to cleanup session layer state. */ - stream_session_delete_notify (&tc->connection); - } + tc = tcp_connection_get (conn_index, vlib_get_thread_index ()); + /* note: the connection may have already disappeared */ + if (PREDICT_FALSE (tc == 0)) + return; + ASSERT (tc->state == TCP_STATE_SYN_RCVD); tc->timers[TCP_TIMER_ESTABLISH] = TCP_TIMER_HANDLE_INVALID; + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + tcp_connection_timers_reset (tc); + /* Start cleanup. Do NOT delete the session until we do the connection + * cleanup. Otherwise, we end up with a dangling session index in the + * tcp connection. */ + tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); +} + +static void +tcp_timer_establish_ao_handler (u32 conn_index) +{ + tcp_connection_t *tc; + + tc = tcp_half_open_connection_get (conn_index); + if (!tc) + return; + + ASSERT (tc->state == TCP_STATE_SYN_SENT); + /* Notify app if we haven't tried to clean this up already */ + if (!(tc->flags & TCP_CONN_HALF_OPEN_DONE)) + session_stream_connect_notify (&tc->connection, 1 /* fail */ ); + + tc->timers[TCP_TIMER_ESTABLISH_AO] = TCP_TIMER_HANDLE_INVALID; tcp_connection_cleanup (tc); } @@ -1180,30 +1324,62 @@ tcp_timer_waitclose_handler (u32 conn_index) return; tc->timers[TCP_TIMER_WAITCLOSE] = TCP_TIMER_HANDLE_INVALID; - /* Session didn't come back with a close(). Send FIN either way - * and switch to LAST_ACK. */ - if (tc->state == TCP_STATE_CLOSE_WAIT) + switch (tc->state) { - if (tc->flags & TCP_CONN_FINSNT) + case TCP_STATE_CLOSE_WAIT: + tcp_connection_timers_reset (tc); + session_transport_closed_notify (&tc->connection); + + if (!(tc->flags & TCP_CONN_FINPNDG)) { - clib_warning ("FIN was sent and still in CLOSE WAIT. Weird!"); + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + break; } - /* Make sure we don't try to send unsent data */ - tcp_connection_timers_reset (tc); + /* Session didn't come back with a close. Send FIN either way + * and switch to LAST_ACK. */ tcp_cong_recovery_off (tc); - tc->snd_una_max = tc->snd_nxt = tc->snd_una; + /* Make sure we don't try to send unsent data */ + tc->snd_nxt = tc->snd_una; tcp_send_fin (tc); - tc->state = TCP_STATE_LAST_ACK; + tcp_connection_set_state (tc, TCP_STATE_LAST_ACK); /* Make sure we don't wait in LAST ACK forever */ tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_2MSL_TIME); /* Don't delete the connection yet */ - return; + break; + case TCP_STATE_FIN_WAIT_1: + tcp_connection_timers_reset (tc); + session_transport_closed_notify (&tc->connection); + if (tc->flags & TCP_CONN_FINPNDG) + { + /* If FIN pending, we haven't sent everything, but we did try. + * Notify session layer that transport is closed. */ + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + tcp_send_reset (tc); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + } + else + { + /* We've sent the fin but no progress. Close the connection and + * to make sure everything is flushed, setup a cleanup timer */ + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + } + break; + case TCP_STATE_LAST_ACK: + case TCP_STATE_CLOSING: + tcp_connection_timers_reset (tc); + tcp_connection_set_state (tc, TCP_STATE_CLOSED); + tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, TCP_CLEANUP_TIME); + session_transport_closed_notify (&tc->connection); + break; + default: + tcp_connection_del (tc); + break; } - - tcp_connection_del (tc); } /* *INDENT-OFF* */ @@ -1215,7 +1391,8 @@ static timer_expiration_handler *timer_expiration_handlers[TCP_N_TIMERS] = tcp_timer_keep_handler, tcp_timer_waitclose_handler, tcp_timer_retransmit_syn_handler, - tcp_timer_establish_handler + tcp_timer_establish_handler, + tcp_timer_establish_ao_handler, }; /* *INDENT-ON* */ @@ -1252,16 +1429,25 @@ tcp_initialize_timer_wheels (tcp_main_t * tm) /* *INDENT-ON* */ } +static void +tcp_initialize_iss_seed (tcp_main_t * tm) +{ + u32 default_seed = random_default_seed (); + u64 time_now = clib_cpu_time_now (); + + tm->iss_seed.first = (u64) random_u32 (&default_seed) << 32; + tm->iss_seed.second = random_u64 (&time_now); +} + static clib_error_t * tcp_main_enable (vlib_main_t * vm) { - tcp_main_t *tm = vnet_get_tcp_main (); vlib_thread_main_t *vtm = vlib_get_thread_main (); + u32 num_threads, n_workers, prealloc_conn_per_wrk; + tcp_connection_t *tc __attribute__ ((unused)); + tcp_main_t *tm = vnet_get_tcp_main (); clib_error_t *error = 0; - u32 num_threads; int thread; - tcp_connection_t *tc __attribute__ ((unused)); - u32 preallocated_connections_per_thread; if ((error = vlib_call_init_function (vm, ip_main_init))) return error; @@ -1283,27 +1469,24 @@ tcp_main_enable (vlib_main_t * vm) num_threads = 1 /* main thread */ + vtm->n_threads; vec_validate (tm->connections, num_threads - 1); + vec_validate (tm->wrk_ctx, num_threads - 1); + n_workers = num_threads == 1 ? 1 : vtm->n_threads; + prealloc_conn_per_wrk = tm->preallocated_connections / n_workers; - /* - * Preallocate connections. Assume that thread 0 won't - * use preallocated threads when running multi-core - */ - if (num_threads == 1) + for (thread = 0; thread < num_threads; thread++) { - thread = 0; - preallocated_connections_per_thread = tm->preallocated_connections; - } - else - { - thread = 1; - preallocated_connections_per_thread = - tm->preallocated_connections / (num_threads - 1); - } - for (; thread < num_threads; thread++) - { - if (preallocated_connections_per_thread) - pool_init_fixed (tm->connections[thread], - preallocated_connections_per_thread); + vec_validate (tm->wrk_ctx[thread].pending_deq_acked, 255); + vec_validate (tm->wrk_ctx[thread].pending_disconnects, 255); + vec_reset_length (tm->wrk_ctx[thread].pending_deq_acked); + vec_reset_length (tm->wrk_ctx[thread].pending_disconnects); + tm->wrk_ctx[thread].vm = vlib_mains[thread]; + + /* + * Preallocate connections. Assume that thread 0 won't + * use preallocated threads when running multi-core + */ + if ((thread > 0 || num_threads == 1) && prealloc_conn_per_wrk) + pool_init_fixed (tm->connections[thread], prealloc_conn_per_wrk); } /* @@ -1323,12 +1506,11 @@ tcp_main_enable (vlib_main_t * vm) clib_spinlock_init (&tm->half_open_lock); } - vec_validate (tm->wrk_ctx, num_threads - 1); tcp_initialize_timer_wheels (tm); + tcp_initialize_iss_seed (tm); - tm->bytes_per_buffer = vlib_buffer_free_list_buffer_size - (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX); - + tm->bytes_per_buffer = vlib_buffer_get_default_data_size (vm); + tm->cc_last_type = TCP_CC_LAST; return error; } @@ -1384,11 +1566,59 @@ tcp_init (vlib_main_t * vm) FIB_PROTOCOL_IP6, tcp6_output_node.index); tcp_api_reference (); + tm->cc_algo_by_name = hash_create_string (0, sizeof (uword)); + tm->tx_pacing = 1; + tm->cc_algo = TCP_CC_NEWRENO; + tm->default_mtu = 1460; + tm->initial_cwnd_multiplier = 0; return 0; } VLIB_INIT_FUNCTION (tcp_init); +uword +unformat_tcp_cc_algo (unformat_input_t * input, va_list * va) +{ + uword *result = va_arg (*va, uword *); + tcp_main_t *tm = &tcp_main; + char *cc_algo_name; + u8 found = 0; + uword *p; + + if (unformat (input, "%s", &cc_algo_name) + && ((p = hash_get_mem (tm->cc_algo_by_name, cc_algo_name)))) + { + *result = *p; + found = 1; + } + + vec_free (cc_algo_name); + return found; +} + +uword +unformat_tcp_cc_algo_cfg (unformat_input_t * input, va_list * va) +{ + tcp_main_t *tm = vnet_get_tcp_main (); + tcp_cc_algorithm_t *cc_alg; + unformat_input_t sub_input; + int found = 0; + + vec_foreach (cc_alg, tm->cc_algos) + { + if (!unformat (input, cc_alg->name)) + continue; + + if (cc_alg->unformat_cfg + && unformat (input, "%U", unformat_vlib_cli_sub_input, &sub_input)) + { + if (cc_alg->unformat_cfg (&sub_input)) + found = 1; + } + } + return found; +} + static clib_error_t * tcp_config_fn (vlib_main_t * vm, unformat_input_t * input) { @@ -1408,6 +1638,18 @@ tcp_config_fn (vlib_main_t * vm, unformat_input_t * input) else if (unformat (input, "max-rx-fifo %U", unformat_memory_size, &tm->max_rx_fifo)) ; + else if (unformat (input, "mtu %d", &tm->default_mtu)) + ; + else if (unformat (input, "initial-cwnd-multiplier %d", + &tm->initial_cwnd_multiplier)) + ; + else if (unformat (input, "no-tx-pacing")) + tm->tx_pacing = 0; + else if (unformat (input, "cc-algo %U", unformat_tcp_cc_algo, + &tm->cc_algo)) + ; + else if (unformat (input, "%U", unformat_tcp_cc_algo_cfg)) + ; else return clib_error_return (0, "unknown input `%U'", format_unformat_error, input); @@ -1436,14 +1678,10 @@ tcp_configure_v4_source_address_range (vlib_main_t * vm, vnet_main_t *vnm = vnet_get_main (); u32 start_host_byte_order, end_host_byte_order; fib_prefix_t prefix; - vnet_sw_interface_t *si; fib_node_index_t fei; u32 fib_index = 0; u32 sw_if_index; int rv; - int vnet_proxy_arp_add_del (ip4_address_t * lo_addr, - ip4_address_t * hi_addr, u32 fib_index, - int is_del); clib_memset (&prefix, 0, sizeof (prefix)); @@ -1472,13 +1710,14 @@ tcp_configure_v4_source_address_range (vlib_main_t * vm, sw_if_index = fib_entry_get_resolving_interface (fei); - /* Enable proxy arp on the interface */ - si = vnet_get_sw_interface (vnm, sw_if_index); - si->flags |= VNET_SW_INTERFACE_FLAG_PROXY_ARP; - /* Configure proxy arp across the range */ rv = vnet_proxy_arp_add_del (start, end, fib_index, 0 /* is_del */ ); + if (rv) + return rv; + + rv = vnet_proxy_arp_enable_disable (vnm, sw_if_index, 1); + if (rv) return rv; @@ -1752,6 +1991,12 @@ tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose) tcp_connection_t _dummy_tc, *dummy_tc = &_dummy_tc; sack_block_t *block; + if (!TCP_SCOREBOARD_TRACE) + { + s = format (s, "scoreboard tracing not enabled"); + return s; + } + if (!tc) return s; @@ -1763,9 +2008,6 @@ tcp_scoreboard_replay (u8 * s, tcp_connection_t * tc, u8 verbose) #if TCP_SCOREBOARD_TRACE trace = tc->sack_sb.trace; trace_len = vec_len (tc->sack_sb.trace); -#else - trace = 0; - trace_len = 0; #endif for (i = 0; i < trace_len; i++)