From 0765d97abef74727c040d2eaf9112865d59f1593 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 18 Mar 2020 21:26:41 +0000 Subject: [PATCH] tcp: explicit use of timer wheel in timer apis Type: refactor Signed-off-by: Florin Coras Change-Id: I06120d43c15b2fa3f437ef235bf5f83ec2beb45e --- src/vnet/tcp/tcp.c | 45 +++++++++++------- src/vnet/tcp/tcp.h | 114 +--------------------------------------------- src/vnet/tcp/tcp_input.c | 50 +++++++++++++------- src/vnet/tcp/tcp_output.c | 44 +++++++++--------- src/vnet/tcp/tcp_timer.h | 105 +++++++++++++++++++++++++++++++++++++++++- src/vnet/tcp/tcp_types.h | 3 ++ 6 files changed, 190 insertions(+), 171 deletions(-) diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c index 2ac938a1b1e..d85225b9172 100644 --- a/src/vnet/tcp/tcp.c +++ b/src/vnet/tcp/tcp.c @@ -208,10 +208,14 @@ tcp_half_open_connection_del (tcp_connection_t * tc) int tcp_half_open_connection_cleanup (tcp_connection_t * tc) { + tcp_worker_ctx_t *wrk; + /* Make sure this is the owning thread */ if (tc->c_thread_index != vlib_get_thread_index ()) return 1; - tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT_SYN); + + wrk = tcp_get_worker (tc->c_thread_index); + tcp_timer_reset (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN); tcp_half_open_connection_del (tc); return 0; } @@ -354,6 +358,8 @@ tcp_program_cleanup (tcp_worker_ctx_t * wrk, tcp_connection_t * tc) void tcp_connection_close (tcp_connection_t * tc) { + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + TCP_EVT (TCP_EVT_CLOSE, tc); /* Send/Program FIN if needed and switch state */ @@ -368,7 +374,8 @@ tcp_connection_close (tcp_connection_t * tc) tcp_connection_timers_reset (tc); tcp_send_fin (tc); tcp_connection_set_state (tc, TCP_STATE_FIN_WAIT_1); - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait1_time); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.finwait1_time); break; case TCP_STATE_ESTABLISHED: /* If closing with unread data, reset the connection */ @@ -379,7 +386,7 @@ tcp_connection_close (tcp_connection_t * tc) tcp_connection_set_state (tc, TCP_STATE_CLOSED); session_transport_closed_notify (&tc->connection); tcp_program_cleanup (tcp_get_worker (tc->c_thread_index), tc); - tcp_worker_stats_inc (tc->c_thread_index, rst_unread, 1); + tcp_worker_stats_inc (wrk, rst_unread, 1); break; } if (!transport_max_tx_dequeue (&tc->connection)) @@ -390,7 +397,8 @@ tcp_connection_close (tcp_connection_t * tc) /* Set a timer in case the peer stops responding. Otherwise the * connection will be stuck here forever. */ ASSERT (tc->timers[TCP_TIMER_WAITCLOSE] == TCP_TIMER_HANDLE_INVALID); - tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait1_time); + tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.finwait1_time); break; case TCP_STATE_CLOSE_WAIT: if (!transport_max_tx_dequeue (&tc->connection)) @@ -398,13 +406,15 @@ tcp_connection_close (tcp_connection_t * tc) tcp_send_fin (tc); tcp_connection_timers_reset (tc); tcp_connection_set_state (tc, TCP_STATE_LAST_ACK); - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.lastack_time); } else tc->flags |= TCP_CONN_FINPNDG; break; case TCP_STATE_FIN_WAIT_1: - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait1_time); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.finwait1_time); break; case TCP_STATE_CLOSED: /* Cleanup should've been programmed already */ @@ -469,11 +479,11 @@ tcp_connection_timers_init (tcp_connection_t * tc) void tcp_connection_timers_reset (tcp_connection_t * tc) { + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); int i; + for (i = 0; i < TCP_N_TIMERS; i++) - { - tcp_timer_reset (tc, i); - } + tcp_timer_reset (&wrk->timer_wheel, tc, i); } #if 0 @@ -956,7 +966,7 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc) tcp_connection_set_state (tc, TCP_STATE_CLOSED); session_transport_closed_notify (&tc->connection); tcp_program_cleanup (wrk, tc); - tcp_workerp_stats_inc (wrk, to_closewait, 1); + tcp_worker_stats_inc (wrk, to_closewait, 1); break; } @@ -969,8 +979,9 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc) session_transport_closed_notify (&tc->connection); /* Make sure we don't wait in LAST ACK forever */ - tcp_timer_set (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time); - tcp_workerp_stats_inc (wrk, to_closewait2, 1); + tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.lastack_time); + tcp_worker_stats_inc (wrk, to_closewait2, 1); /* Don't delete the connection yet */ break; @@ -992,21 +1003,21 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc) tcp_program_cleanup (wrk, tc); } session_transport_closed_notify (&tc->connection); - tcp_workerp_stats_inc (wrk, to_finwait1, 1); + tcp_worker_stats_inc (wrk, to_finwait1, 1); break; case TCP_STATE_LAST_ACK: tcp_connection_timers_reset (tc); tcp_connection_set_state (tc, TCP_STATE_CLOSED); session_transport_closed_notify (&tc->connection); tcp_program_cleanup (wrk, tc); - tcp_workerp_stats_inc (wrk, to_lastack, 1); + tcp_worker_stats_inc (wrk, to_lastack, 1); break; case TCP_STATE_CLOSING: tcp_connection_timers_reset (tc); tcp_connection_set_state (tc, TCP_STATE_CLOSED); session_transport_closed_notify (&tc->connection); tcp_program_cleanup (wrk, tc); - tcp_workerp_stats_inc (wrk, to_closing, 1); + tcp_worker_stats_inc (wrk, to_closing, 1); break; case TCP_STATE_FIN_WAIT_2: tcp_send_reset (tc); @@ -1014,7 +1025,7 @@ tcp_timer_waitclose_handler (tcp_connection_t * tc) tcp_connection_set_state (tc, TCP_STATE_CLOSED); session_transport_closed_notify (&tc->connection); tcp_program_cleanup (wrk, tc); - tcp_workerp_stats_inc (wrk, to_finwait2, 1); + tcp_worker_stats_inc (wrk, to_finwait2, 1); break; case TCP_STATE_TIME_WAIT: tcp_connection_set_state (tc, TCP_STATE_CLOSED); @@ -1186,7 +1197,7 @@ tcp_expired_timers_dispatch (u32 * expired_timers) wrk = tcp_get_worker (thread_index); n_expired = vec_len (expired_timers); - tcp_workerp_stats_inc (wrk, timer_expirations, n_expired); + tcp_worker_stats_inc (wrk, timer_expirations, n_expired); n_left = clib_fifo_elts (wrk->pending_timers); /* diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h index 708d7566eb2..f8c8cb342fe 100644 --- a/src/vnet/tcp/tcp.h +++ b/src/vnet/tcp/tcp.h @@ -115,17 +115,14 @@ typedef struct tcp_worker_ctx_ tcp_cleanup_req_t *pending_cleanups; /** worker timer wheel */ - tw_timer_wheel_16t_2w_512sl_t timer_wheel; + tcp_timer_wheel_t timer_wheel; CLIB_CACHE_LINE_ALIGN_MARK (cacheline2); tcp_wrk_stats_t stats; } tcp_worker_ctx_t; -#define tcp_worker_stats_inc(_ti,_stat,_val) \ - tcp_main.wrk_ctx[_ti].stats._stat += _val - -#define tcp_workerp_stats_inc(_wrk,_stat,_val) \ +#define tcp_worker_stats_inc(_wrk,_stat,_val) \ _wrk->stats._stat += _val typedef struct tcp_iss_seed_ @@ -356,113 +353,6 @@ format_function_t format_tcp_rcv_sacks; format_function_t format_tcp_connection; format_function_t format_tcp_connection_id; -always_inline void -tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval) -{ - ASSERT (tc->c_thread_index == vlib_get_thread_index ()); - ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID); - tc->timers[timer_id] = - tw_timer_start_16t_2w_512sl (&tcp_main. - wrk_ctx[tc->c_thread_index].timer_wheel, - tc->c_c_index, timer_id, interval); -} - -always_inline void -tcp_timer_reset (tcp_connection_t * tc, u8 timer_id) -{ - ASSERT (tc->c_thread_index == vlib_get_thread_index ()); - if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID) - return; - - tw_timer_stop_16t_2w_512sl (&tcp_main. - wrk_ctx[tc->c_thread_index].timer_wheel, - tc->timers[timer_id]); - tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID; -} - -always_inline void -tcp_timer_update (tcp_connection_t * tc, u8 timer_id, u32 interval) -{ - ASSERT (tc->c_thread_index == vlib_get_thread_index ()); - if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID) - tw_timer_update_16t_2w_512sl (&tcp_main. - wrk_ctx[tc->c_thread_index].timer_wheel, - tc->timers[timer_id], interval); - else - tc->timers[timer_id] = - tw_timer_start_16t_2w_512sl (&tcp_main. - wrk_ctx[tc->c_thread_index].timer_wheel, - tc->c_c_index, timer_id, interval); -} - -always_inline void -tcp_retransmit_timer_set (tcp_connection_t * tc) -{ - ASSERT (tc->snd_una != tc->snd_una_max); - tcp_timer_set (tc, TCP_TIMER_RETRANSMIT, - clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); -} - -always_inline void -tcp_retransmit_timer_reset (tcp_connection_t * tc) -{ - tcp_timer_reset (tc, TCP_TIMER_RETRANSMIT); -} - -always_inline void -tcp_retransmit_timer_force_update (tcp_connection_t * tc) -{ - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, - clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); -} - -always_inline void -tcp_persist_timer_set (tcp_connection_t * tc) -{ - /* Reuse RTO. It's backed off in handler */ - tcp_timer_set (tc, TCP_TIMER_PERSIST, - clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); -} - -always_inline void -tcp_persist_timer_update (tcp_connection_t * tc) -{ - u32 interval; - - if (seq_leq (tc->snd_una, tc->snd_congestion + tc->burst_acked)) - interval = 1; - else - interval = clib_max (tc->rto * TCP_TO_TIMER_TICK, 1); - - tcp_timer_update (tc, TCP_TIMER_PERSIST, interval); -} - -always_inline void -tcp_persist_timer_reset (tcp_connection_t * tc) -{ - tcp_timer_reset (tc, TCP_TIMER_PERSIST); -} - -always_inline void -tcp_retransmit_timer_update (tcp_connection_t * tc) -{ - if (tc->snd_una == tc->snd_nxt) - { - tcp_retransmit_timer_reset (tc); - if (tc->snd_wnd < tc->snd_mss) - tcp_persist_timer_update (tc); - } - else - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, - clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); -} - -always_inline u8 -tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer) -{ - return tc->timers[timer] != TCP_TIMER_HANDLE_INVALID; -} - #define tcp_validate_txf_size(_tc, _a) \ ASSERT(_tc->state != TCP_STATE_ESTABLISHED \ || transport_max_tx_dequeue (&_tc->connection) >= _a) diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c index e27cffb9444..713e11fd896 100755 --- a/src/vnet/tcp/tcp_input.c +++ b/src/vnet/tcp/tcp_input.c @@ -583,7 +583,7 @@ tcp_handle_postponed_dequeues (tcp_worker_ctx_t * wrk) /* If everything has been acked, stop retransmit timer * otherwise update. */ - tcp_retransmit_timer_update (tc); + tcp_retransmit_timer_update (&wrk->timer_wheel, tc); /* Update pacer based on our new cwnd estimate */ tcp_connection_tx_pacer_update (tc); @@ -628,12 +628,18 @@ tcp_update_snd_wnd (tcp_connection_t * tc, u32 seq, u32 ack, u32 snd_wnd) /* Set persist timer if not set and we just got 0 wnd */ if (!tcp_timer_is_active (tc, TCP_TIMER_PERSIST) && !tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)) - tcp_persist_timer_set (tc); + { + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + tcp_persist_timer_set (&wrk->timer_wheel, tc); + } } else { if (PREDICT_FALSE (tcp_timer_is_active (tc, TCP_TIMER_PERSIST))) - tcp_persist_timer_reset (tc); + { + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + tcp_persist_timer_reset (&wrk->timer_wheel, tc); + } if (PREDICT_FALSE (tcp_is_descheduled (tc))) tcp_reschedule (tc); @@ -1137,7 +1143,8 @@ tcp_rcv_fin (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, vlib_buffer_t * b, * in CLOSE-WAIT, set timer (reuse WAITCLOSE). */ tcp_connection_set_state (tc, TCP_STATE_CLOSE_WAIT); tcp_program_disconnect (wrk, tc); - tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_WAITCLOSE, + tcp_cfg.closewait_time); TCP_EVT (TCP_EVT_FIN_RCVD, tc); *error = TCP_ERROR_FIN_RCVD; } @@ -1356,7 +1363,8 @@ in_order: if (tcp_can_delack (tc)) { if (!tcp_timer_is_active (tc, TCP_TIMER_DELACK)) - tcp_timer_set (tc, TCP_TIMER_DELACK, tcp_cfg.delack_time); + tcp_timer_set (&wrk->timer_wheel, tc, TCP_TIMER_DELACK, + tcp_cfg.delack_time); goto done; } @@ -2224,7 +2232,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tc0->snd_wl2 = vnet_buffer (b0)->tcp.ack_number; /* Reset SYN-ACK retransmit and SYN_RCV establish timers */ - tcp_retransmit_timer_reset (tc0); + tcp_retransmit_timer_reset (&wrk->timer_wheel, tc0); if (session_stream_accept_notify (&tc0->connection)) { error0 = TCP_ERROR_MSG_QUEUE_FULL; @@ -2258,7 +2266,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_send_fin (tc0); /* If a fin was received and data was acked extend wait */ else if ((tc0->flags & TCP_CONN_FINRCVD) && tc0->bytes_acked) - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); } /* If FIN is ACKed */ @@ -2281,7 +2289,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_connection_set_state (tc0, TCP_STATE_FIN_WAIT_2); /* Enable waitclose because we're willing to wait for peer's * FIN but not indefinitely. */ - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.finwait2_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.finwait2_time); /* Don't try to deq the FIN acked */ if (tc0->burst_acked > 1) @@ -2314,7 +2323,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_send_fin (tc0); tcp_connection_timers_reset (tc0); tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.lastack_time); break; case TCP_STATE_CLOSING: /* In addition to the processing for the ESTABLISHED state, if @@ -2328,7 +2338,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_connection_timers_reset (tc0); tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); session_transport_closed_notify (&tc0->connection); goto drop; @@ -2372,7 +2383,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, goto drop; tcp_program_ack (tc0); - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); goto drop; break; @@ -2414,7 +2426,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_program_ack (tc0); tcp_connection_set_state (tc0, TCP_STATE_CLOSE_WAIT); tcp_program_disconnect (wrk, tc0); - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.closewait_time); break; case TCP_STATE_SYN_RCVD: /* Send FIN-ACK, enter LAST-ACK and because the app was not @@ -2424,7 +2437,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tc0->rcv_nxt += 1; tcp_send_fin (tc0); tcp_connection_set_state (tc0, TCP_STATE_LAST_ACK); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.lastack_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.lastack_time); break; case TCP_STATE_CLOSE_WAIT: case TCP_STATE_CLOSING: @@ -2440,7 +2454,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, * sending it. Since we already received a fin, do not wait * for too long. */ tc0->flags |= TCP_CONN_FINRCVD; - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time); } else @@ -2448,7 +2462,7 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tcp_connection_set_state (tc0, TCP_STATE_CLOSING); tcp_program_ack (tc0); /* Wait for ACK for our FIN but not forever */ - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.closing_time); } break; @@ -2457,7 +2471,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, tc0->rcv_nxt += 1; tcp_connection_set_state (tc0, TCP_STATE_TIME_WAIT); tcp_connection_timers_reset (tc0); - tcp_timer_set (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_set (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); tcp_program_ack (tc0); session_transport_closed_notify (&tc0->connection); break; @@ -2465,7 +2480,8 @@ tcp46_rcv_process_inline (vlib_main_t * vm, vlib_node_runtime_t * node, /* Remain in the TIME-WAIT state. Restart the time-wait * timeout. */ - tcp_timer_update (tc0, TCP_TIMER_WAITCLOSE, tcp_cfg.timewait_time); + tcp_timer_update (&wrk->timer_wheel, tc0, TCP_TIMER_WAITCLOSE, + tcp_cfg.timewait_time); break; } error0 = TCP_ERROR_FIN_RCVD; diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index aeeffa726c3..a68d5269abd 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -808,12 +808,12 @@ tcp_send_syn (tcp_connection_t * tc) * Setup retransmit and establish timers before requesting buffer * such that we can return if we've ran out. */ - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, tc->rto * TCP_TO_TIMER_TICK); if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, 1); return; } @@ -839,11 +839,11 @@ tcp_send_synack (tcp_connection_t * tc) vlib_buffer_t *b; u32 bi; - tcp_retransmit_timer_force_update (tc); + tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc); if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1); return; } @@ -874,7 +874,7 @@ tcp_send_fin (tcp_connection_t * tc) if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { /* Out of buffers so program fin retransmit ASAP */ - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1); if (fin_snt) tc->snd_nxt += 1; else @@ -887,7 +887,7 @@ tcp_send_fin (tcp_connection_t * tc) if ((tc->flags & TCP_CONN_SNDACK) && !tc->pending_dupacks) tc->flags &= ~TCP_CONN_SNDACK; - tcp_retransmit_timer_force_update (tc); + tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc); b = vlib_get_buffer (vm, bi); tcp_init_buffer (vm, b); tcp_make_fin (tc, b); @@ -1001,7 +1001,8 @@ tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) } if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT))) { - tcp_retransmit_timer_set (tc); + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + tcp_retransmit_timer_set (&wrk->timer_wheel, tc); tc->rto_boff = 0; } tcp_trajectory_add_start (b, 3); @@ -1252,7 +1253,7 @@ tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk, tc->bytes_retrans += n_bytes; tc->segs_retrans += 1; - tcp_workerp_stats_inc (wrk, rxt_segs, 1); + tcp_worker_stats_inc (wrk, rxt_segs, 1); TCP_EVT (TCP_EVT_CC_RTX, tc, offset, n_bytes); return n_bytes; @@ -1304,7 +1305,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) vlib_buffer_t *b = 0; u32 bi, n_bytes; - tcp_workerp_stats_inc (wrk, tr_events, 1); + tcp_worker_stats_inc (wrk, tr_events, 1); /* Should be handled by a different handler */ if (PREDICT_FALSE (tc->state == TCP_STATE_SYN_SENT)) @@ -1357,7 +1358,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) session_transport_closed_notify (&tc->connection); tcp_connection_timers_reset (tc); tcp_program_cleanup (wrk, tc); - tcp_workerp_stats_inc (wrk, tr_abort, 1); + tcp_worker_stats_inc (wrk, tr_abort, 1); return; } @@ -1373,7 +1374,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, n_bytes, &b); if (!n_bytes) { - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1); return; } @@ -1381,7 +1382,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); - tcp_retransmit_timer_force_update (tc); + tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc); tc->rto_boff += 1; if (tc->rto_boff == 1) @@ -1409,13 +1410,13 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) tcp_connection_set_state (tc, TCP_STATE_CLOSED); tcp_connection_timers_reset (tc); tcp_program_cleanup (wrk, tc); - tcp_workerp_stats_inc (wrk, tr_abort, 1); + tcp_worker_stats_inc (wrk, tr_abort, 1); return; } if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT, 1); return; } @@ -1423,7 +1424,7 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) if (tc->rto_boff > TCP_RTO_SYN_RETRIES) tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); - tcp_retransmit_timer_force_update (tc); + tcp_retransmit_timer_force_update (&wrk->timer_wheel, tc); b = vlib_get_buffer (vm, bi); tcp_init_buffer (vm, b); @@ -1478,7 +1479,7 @@ tcp_timer_retransmit_syn_handler (tcp_connection_t * tc) if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1); + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, 1); return; } @@ -1498,7 +1499,7 @@ tcp_timer_retransmit_syn_handler (tcp_connection_t * tc) tcp_push_ip_hdr (wrk, tc, b); tcp_enqueue_to_ip_lookup (wrk, b, bi, tc->c_is_ip4, tc->c_fib_index); - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, + tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, tc->rto * TCP_TO_TIMER_TICK); } @@ -1529,7 +1530,7 @@ tcp_timer_persist_handler (tcp_connection_t * tc) * next time */ if (!available_bytes) { - tcp_persist_timer_set (tc); + tcp_persist_timer_set (&wrk->timer_wheel, tc); return; } @@ -1545,7 +1546,7 @@ tcp_timer_persist_handler (tcp_connection_t * tc) */ if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { - tcp_persist_timer_set (tc); + tcp_persist_timer_set (&wrk->timer_wheel, tc); return; } @@ -1576,7 +1577,7 @@ tcp_timer_persist_handler (tcp_connection_t * tc) tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); /* Just sent new data, enable retransmit */ - tcp_retransmit_timer_update (tc); + tcp_retransmit_timer_update (&wrk->timer_wheel, tc); return; @@ -2171,9 +2172,6 @@ tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0, } } - if (!TCP_ALWAYS_ACK) - tcp_timer_reset (tc0, TCP_TIMER_DELACK); - tc0->segs_out += 1; } diff --git a/src/vnet/tcp/tcp_timer.h b/src/vnet/tcp/tcp_timer.h index 08516b7eec2..d34fdcb146d 100644 --- a/src/vnet/tcp/tcp_timer.h +++ b/src/vnet/tcp/tcp_timer.h @@ -15,8 +15,109 @@ #ifndef __included_tcp_timer_h__ #define __included_tcp_timer_h__ -#include -#include +#include + +always_inline void +tcp_timer_set (tcp_timer_wheel_t * tw, tcp_connection_t * tc, u8 timer_id, + u32 interval) +{ + ASSERT (tc->c_thread_index == vlib_get_thread_index ()); + ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID); + tc->timers[timer_id] = tw_timer_start_16t_2w_512sl (tw, tc->c_c_index, + timer_id, interval); +} + +always_inline void +tcp_timer_reset (tcp_timer_wheel_t * tw, tcp_connection_t * tc, u8 timer_id) +{ + ASSERT (tc->c_thread_index == vlib_get_thread_index ()); + if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID) + return; + + tw_timer_stop_16t_2w_512sl (tw, tc->timers[timer_id]); + tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID; +} + +always_inline void +tcp_timer_update (tcp_timer_wheel_t * tw, tcp_connection_t * tc, u8 timer_id, + u32 interval) +{ + ASSERT (tc->c_thread_index == vlib_get_thread_index ()); + if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID) + tw_timer_update_16t_2w_512sl (tw, tc->timers[timer_id], interval); + else + tc->timers[timer_id] = tw_timer_start_16t_2w_512sl (tw, tc->c_c_index, + timer_id, interval); +} + +always_inline void +tcp_retransmit_timer_set (tcp_timer_wheel_t * tw, tcp_connection_t * tc) +{ + ASSERT (tc->snd_una != tc->snd_una_max); + tcp_timer_set (tw, tc, TCP_TIMER_RETRANSMIT, + clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); +} + +always_inline void +tcp_retransmit_timer_reset (tcp_timer_wheel_t * tw, tcp_connection_t * tc) +{ + tcp_timer_reset (tw, tc, TCP_TIMER_RETRANSMIT); +} + +always_inline void +tcp_retransmit_timer_force_update (tcp_timer_wheel_t * tw, + tcp_connection_t * tc) +{ + tcp_timer_update (tw, tc, TCP_TIMER_RETRANSMIT, + clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); +} + +always_inline void +tcp_persist_timer_set (tcp_timer_wheel_t * tw, tcp_connection_t * tc) +{ + /* Reuse RTO. It's backed off in handler */ + tcp_timer_set (tw, tc, TCP_TIMER_PERSIST, + clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); +} + +always_inline void +tcp_persist_timer_update (tcp_timer_wheel_t * tw, tcp_connection_t * tc) +{ + u32 interval; + + if (seq_leq (tc->snd_una, tc->snd_congestion + tc->burst_acked)) + interval = 1; + else + interval = clib_max (tc->rto * TCP_TO_TIMER_TICK, 1); + + tcp_timer_update (tw, tc, TCP_TIMER_PERSIST, interval); +} + +always_inline void +tcp_persist_timer_reset (tcp_timer_wheel_t * tw, tcp_connection_t * tc) +{ + tcp_timer_reset (tw, tc, TCP_TIMER_PERSIST); +} + +always_inline void +tcp_retransmit_timer_update (tcp_timer_wheel_t * tw, tcp_connection_t * tc) +{ + if (tc->snd_una == tc->snd_nxt) + { + tcp_retransmit_timer_reset (tw, tc); + if (tc->snd_wnd < tc->snd_mss) + tcp_persist_timer_update (tw, tc); + } + else + tcp_timer_update (tw, tc, TCP_TIMER_RETRANSMIT, + clib_max (tc->rto * TCP_TO_TIMER_TICK, 1)); +} + +always_inline u8 +tcp_timer_is_active (tcp_connection_t * tc, tcp_timers_e timer) +{ + return tc->timers[timer] != TCP_TIMER_HANDLE_INVALID; +} #endif /* __included_tcp_timer_h__ */ diff --git a/src/vnet/tcp/tcp_types.h b/src/vnet/tcp/tcp_types.h index ccb7ae86aed..cc4f4b3df3a 100644 --- a/src/vnet/tcp/tcp_types.h +++ b/src/vnet/tcp/tcp_types.h @@ -20,6 +20,7 @@ #include #include #include +#include #define TCP_TICK 0.001 /**< TCP tick period (s) */ #define THZ (u32) (1/TCP_TICK) /**< TCP tick frequency */ @@ -440,6 +441,8 @@ tcp_get_connection_from_transport (transport_connection_t * tconn) return (tcp_connection_t *) tconn; } +typedef tw_timer_wheel_16t_2w_512sl_t tcp_timer_wheel_t; + #endif /* SRC_VNET_TCP_TCP_TYPES_H_ */ /* -- 2.16.6