X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp_input.c;h=b8c889ee1cc7f3c901ae83e9132c1f314377fea9;hb=d2f5174dd045da53395939cd55a0f6e2821f6dcf;hp=e27cffb9444b6b0dac213147d7b422041d06cab1;hpb=999840cf805f26a490e8e6b8acc1fe7a7c21a181;p=vpp.git diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c old mode 100755 new mode 100644 index e27cffb9444..b8c889ee1cc --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -426,8 +426,6 @@ acceptable: * Note that although the original article, srtt and rttvar are scaled * to minimize round-off errors, here we don't. Instead, we rely on * better precision time measurements. - * - * TODO support us rtt resolution */ static void tcp_estimate_rtt (tcp_connection_t * tc, u32 mrtt) @@ -452,16 +450,28 @@ tcp_estimate_rtt (tcp_connection_t * tc, u32 mrtt) } } +static inline void +tcp_estimate_rtt_us (tcp_connection_t * tc, f64 mrtt) +{ + tc->mrtt_us = tc->mrtt_us + (mrtt - tc->mrtt_us) * 0.125; +} + /** - * Update RTT estimate and RTO timer + * Update rtt estimate + * + * We have potentially three sources of rtt measurements: * - * Measure RTT: We have two sources of RTT measurements: TSOPT and ACK - * timing. Middle boxes are known to fiddle with TCP options so we - * should give higher priority to ACK timing. + * TSOPT difference between current and echoed timestamp. It has ms + * precision and can be computed per ack + * ACK timing one sequence number is tracked per rtt with us (micro second) + * precision. + * rate sample if enabled, all outstanding bytes are tracked with us + * precision. Every ack and sack are a rtt sample * - * This should be called only if previously sent bytes have been acked. + * Middle boxes are known to fiddle with TCP options so we give higher + * priority to ACK timing. * - * return 1 if valid rtt 0 otherwise + * For now, rate sample rtts are only used under congestion. */ static int tcp_update_rtt (tcp_connection_t * tc, tcp_rate_sample_t * rs, u32 ack) @@ -473,19 +483,19 @@ tcp_update_rtt (tcp_connection_t * tc, tcp_rate_sample_t * rs, u32 ack) if (tcp_in_cong_recovery (tc)) { /* Accept rtt estimates for samples that have not been retransmitted */ - if ((tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE) - && !(rs->flags & TCP_BTS_IS_RXT)) - { - mrtt = rs->rtt_time * THZ; - goto estimate_rtt; - } - goto done; + if (!(tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE) + || (rs->flags & TCP_BTS_IS_RXT)) + goto done; + if (rs->rtt_time) + tcp_estimate_rtt_us (tc, rs->rtt_time); + mrtt = rs->rtt_time * THZ; + goto estimate_rtt; } if (tc->rtt_ts && seq_geq (ack, tc->rtt_seq)) { f64 sample = tcp_time_now_us (tc->c_thread_index) - tc->rtt_ts; - tc->mrtt_us = tc->mrtt_us + (sample - tc->mrtt_us) * 0.125; + tcp_estimate_rtt_us (tc, sample); mrtt = clib_max ((u32) (sample * THZ), 1); /* Allow measuring of a new RTT */ tc->rtt_ts = 0; @@ -583,7 +593,7 @@ tcp_handle_postponed_dequeues (tcp_worker_ctx_t * wrk) /* If everything has been acked, stop retransmit timer * otherwise update. */ - tcp_retransmit_timer_update (tc); + tcp_retransmit_timer_update (&wrk->timer_wheel, tc); /* Update pacer based on our new cwnd estimate */ tcp_connection_tx_pacer_update (tc); @@ -628,12 +638,18 @@ tcp_update_snd_wnd (tcp_connection_t * tc, u32 seq, u32 ack, u32 snd_wnd) /* Set persist timer if not set and we just got 0 wnd */ if (!tcp_timer_is_active (tc, TCP_TIMER_PERSIST) && !tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)) - tcp_persist_timer_set (tc); + { + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + tcp_persist_timer_set (&wrk->timer_wheel, tc); + } } else { if (PREDICT_FALSE (tcp_timer_is_active (tc, TCP_TIMER_PERSIST))) - tcp_persist_timer_reset (tc); + { + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + tcp_persist_timer_reset (&wrk->timer_wheel, tc); + } if (PREDICT_FALSE (tcp_is_descheduled (tc))) tcp_reschedule (tc); @@ -1042,10 +1058,11 @@ process_ack: if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE) tcp_bt_sample_delivery_rate (tc, &rs); - if (tc->bytes_acked) + if (tc->bytes_acked + tc->sack_sb.last_sacked_bytes) { - tcp_program_dequeue (wrk, tc); tcp_update_rtt (tc, &rs, vnet_buffer (b)->tcp.ack_number); + if (tc->bytes_acked) + tcp_program_dequeue (wrk, tc); } TCP_EVT (TCP_EVT_ACK_RCVD, tc); @@ -1137,7 +1154,8 @@ tcp_rcv_fin (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t * b, * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */ tcp_connection_set_state (tc, TCP_STATE_CLOSE_WAIT); tcp_program_disconnect (wrk, tc); - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.closewait_time); TCP_EVT (TCP_EVT_FIN_RCVD, tc); *error = TCP_ERROR_FIN_RCVD; } @@ -1245,26 +1263,6 @@ tcp_session_enqueue_ooo (tcp_connection_t * tc, vlib_buffer_t * b, return TCP_ERROR_ENQUEUED_OOO; } -/** - * Check if ACK could be delayed. If ack can be delayed, it should return - * true for a full frame. If we're always acking return 0. - */ -always_inline int -tcp_can_delack (tcp_connection_t * tc) -{ - /* Send ack if ... */ - if (TCP_ALWAYS_ACK - /* just sent a rcv wnd 0 - || (tc->flags & TCP_CONN_SENT_RCV_WND0) != 0 */ - /* constrained to send ack */ - || (tc->flags & TCP_CONN_SNDACK) != 0 - /* we're almost out of tx wnd */ - || tcp_available_cc_snd_space (tc) < 4 * tc->snd_mss) - return 0; - - return 1; -} - static int tcp_buffer_discard_bytes (vlib_buffer_t * b, u32 n_bytes_to_drop) { @@ -1353,13 +1351,6 @@ in_order: /* In order data, enqueue. Fifo figures out by itself if any out-of-order * segments can be enqueued after fifo tail offset changes. */ error = tcp_session_enqueue_data (tc, b, n_data_bytes); - if (tcp_can_delack (tc)) - { - if (!tcp_timer_is_active (tc, TCP_TIMER_DELACK)) - tcp_timer_set (tc, TCP_TIMER_DELACK, tcp_cfg.delack_time); - goto done; - } - tcp_program_ack (tc); done: @@ -1940,11 +1931,6 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, new_tc0->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID; new_tc0->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX]; - /* If this is not the owning thread, wait for syn retransmit to - * expire and cleanup then */ - if (tcp_half_open_connection_cleanup (tc0)) - tc0->flags |= TCP_CONN_HALF_OPEN_DONE; - if (tcp_opts_tstamp (&new_tc0->rcv_opts)) { new_tc0->tsval_recent = new_tc0->rcv_opts.tsval; @@ -1977,12 +1963,13 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* Notify app that we have connection. If session layer can't * allocate session send reset */ - if (session_stream_connect_notify (&new_tc0->connection, 0)) + if (session_stream_connect_notify (&new_tc0->connection, + SESSION_E_NONE)) { tcp_send_reset_w_pkt (new_tc0, b0, my_thread_index, is_ip4); tcp_connection_cleanup (new_tc0); error0 = TCP_ERROR_CREATE_SESSION_FAIL; - goto drop; + goto cleanup_ho; } new_tc0->tx_fifo_size = @@ -1998,13 +1985,14 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, new_tc0->state = TCP_STATE_SYN_RCVD; /* Notify app that we have connection */ - if (session_stream_connect_notify (&new_tc0->connection, 0)) + if (session_stream_connect_notify (&new_tc0->connection, + SESSION_E_NONE)) { tcp_connection_cleanup (new_tc0); tcp_send_reset_w_pkt (tc0, b0, my_thread_index, is_ip4); TCP_EVT (TCP_EVT_RST_SENT, tc0); error0 = TCP_ERROR_CREATE_SESSION_FAIL; - goto drop; + goto cleanup_ho; } new_tc0->tx_fifo_size = @@ -2013,7 +2001,7 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_init_snd_vars (new_tc0); tcp_send_synack (new_tc0); error0 = TCP_ERROR_SYNS_RCVD; - goto drop; + goto cleanup_ho; } if (!(new_tc0->cfg_flags & TCP_CFG_F_NO_TSO)) @@ -2034,6 +2022,13 @@ tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_send_ack (new_tc0); } + cleanup_ho: + + /* If this is not the owning thread, wait for syn retransmit to + * expire and cleanup then */ + if (tcp_half_open_connection_cleanup (tc0)) + tc0->flags |= TCP_CONN_HALF_OPEN_DONE; + drop: tcp_inc_counter (syn_sent, error0, 1); @@ -2224,7 +2219,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tc0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number; /* Reset SYN-ACK retransmit and SYN_RCV establish timers */ - tcp_retransmit_timer_reset (tc0); + tcp_retransmit_timer_reset (&wrk->timer_wheel, tc0); if (session_stream_accept_notify (&tc0->connection)) { error0 = TCP_ERROR_MSG_QUEUE_FULL; @@ -2258,7 +2253,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_send_fin (tc0); /* If a fin was received and data was acked extend wait */ else if ((tc0->flags & TCP_CONN_FINRCVD) && tc0->bytes_acked) - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); } /* If FIN is ACKed */ @@ -2281,7 +2276,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_connection_set_state (tc0, TCP_STATE_FIN_WAIT_2); /* Enable waitclose because we're willing to wait for peer's * FIN but not indefinitely. */ - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait2_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.finwait2_time); /* Don't try to deq the FIN acked */ if (tc0->burst_acked > 1) @@ -2314,7 +2310,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_send_fin (tc0); tcp_connection_timers_reset (tc0); tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.lastack_time); break; case TCP_STATE_CLOSING: /* In addition to the processing for the ESTABLISHED state, if @@ -2328,7 +2325,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_connection_timers_reset (tc0); tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); session_transport_closed_notify (&tc0->connection); goto drop; @@ -2372,7 +2370,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, goto drop; tcp_program_ack (tc0); - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); goto drop; break; @@ -2414,7 +2413,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_program_ack (tc0); tcp_connection_set_state (tc0, TCP_STATE_CLOSE_WAIT); tcp_program_disconnect (wrk, tc0); - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.closewait_time); break; case TCP_STATE_SYN_RCVD: /* Send FIN-ACK, enter LAST-ACK and because the app was not @@ -2424,7 +2424,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tc0->rcv_nxt += 1; tcp_send_fin (tc0); tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.lastack_time); break; case TCP_STATE_CLOSE_WAIT: case TCP_STATE_CLOSING: @@ -2440,7 +2441,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, * sending it. Since we already received a fin, do not wait * for too long. */ tc0->flags |= TCP_CONN_FINRCVD; - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); } else @@ -2448,7 +2449,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_connection_set_state (tc0, TCP_STATE_CLOSING); tcp_program_ack (tc0); /* Wait for ACK for our FIN but not forever */ - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closing_time); } break; @@ -2457,7 +2458,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tc0->rcv_nxt += 1; tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT); tcp_connection_timers_reset (tc0); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); tcp_program_ack (tc0); session_transport_closed_notify (&tc0->connection); break; @@ -2465,7 +2467,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* Remain in the TIME-WAIT state. Restart the time-wait * timeout. */ - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); break; } error0 = TCP_ERROR_FIN_RCVD; @@ -2637,10 +2640,11 @@ tcp46_listen_inline (vlib_main_t * vm, vlib_node_runtime_t * node, child->tx_fifo_size = transport_tx_fifo_size (&child->connection); - tcp_send_synack (child); - + /* This initializes elog track, must be done before synack */ TCP_EVT (TCP_EVT_SYN_RCVD, child, 1); + tcp_send_synack (child); + done: if (PREDICT_FALSE (b->flags & VLIB_BUFFER_IS_TRACED)) @@ -2818,11 +2822,9 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_main_t *tm = vnet_get_tcp_main (); vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; u16 nexts[VLIB_FRAME_SIZE], *next; - vlib_node_runtime_t *error_node; tcp_set_time_now (tcp_get_worker (thread_index)); - error_node = vlib_node_get_runtime (vm, tcp_node_index (input, is_ip4)); from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; vlib_get_buffers (vm, from, bufs, n_left_from); @@ -2858,8 +2860,8 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index; vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index; - tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], error_node); - tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], error_node); + tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], node); + tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], node); } else { @@ -2867,24 +2869,24 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { ASSERT (tcp_lookup_is_valid (tc0, b[0], tcp_buffer_hdr (b[0]))); vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index; - tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], error_node); + tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], node); } else { tcp_input_set_error_next (tm, &next[0], &error0, is_ip4); - b[0]->error = error_node->errors[error0]; + b[0]->error = node->errors[error0]; } if (PREDICT_TRUE (tc1 != 0)) { ASSERT (tcp_lookup_is_valid (tc1, b[1], tcp_buffer_hdr (b[1]))); vnet_buffer (b[1])->tcp.connection_index = tc1->c_c_index; - tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], error_node); + tcp_input_dispatch_buffer (tm, tc1, b[1], &next[1], node); } else { tcp_input_set_error_next (tm, &next[1], &error1, is_ip4); - b[1]->error = error_node->errors[error1]; + b[1]->error = node->errors[error1]; } } @@ -2910,12 +2912,12 @@ tcp46_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, { ASSERT (tcp_lookup_is_valid (tc0, b[0], tcp_buffer_hdr (b[0]))); vnet_buffer (b[0])->tcp.connection_index = tc0->c_c_index; - tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], error_node); + tcp_input_dispatch_buffer (tm, tc0, b[0], &next[0], node); } else { tcp_input_set_error_next (tm, &next[0], &error0, is_ip4); - b[0]->error = error_node->errors[error0]; + b[0]->error = node->errors[error0]; } b += 1;