X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp_output.c;h=5a5c7fe4b8bf7b31582b316a19cda9af0a4575fe;hb=679b78f7155b18efa690b0d2a4a0b51c00090522;hp=5f48d4101262ac975f50a4018b4a2fdd7c7ed097;hpb=54ddf435338ce035c1a92199acd5a3b217db177b;p=vpp.git diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 5f48d410126..5a5c7fe4b8b 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Cisco and/or its affiliates. + * Copyright (c) 2016-2019 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -14,12 +14,8 @@ */ #include -#include #include -vlib_node_registration_t tcp4_output_node; -vlib_node_registration_t tcp6_output_node; - typedef enum _tcp_output_next { TCP_OUTPUT_NEXT_DROP, @@ -53,9 +49,11 @@ typedef struct tcp_connection_t tcp_connection; } tcp_tx_trace_t; +#ifndef CLIB_MARCH_VARIANT u16 dummy_mtu = 1460; +#endif /* CLIB_MARCH_VARIANT */ -u8 * +static u8 * format_tcp_tx_trace (u8 * s, va_list * args) { CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); @@ -71,6 +69,7 @@ format_tcp_tx_trace (u8 * s, va_list * args) return s; } +#ifndef CLIB_MARCH_VARIANT static u8 tcp_window_compute_scale (u32 window) { @@ -173,7 +172,7 @@ tcp_update_rcv_wnd (tcp_connection_t * tc) /* Make sure we have a multiple of rcv_wscale */ if (wnd && tc->rcv_wscale) { - wnd &= ~(1 << tc->rcv_wscale); + wnd &= ~((1 << tc->rcv_wscale) - 1); if (wnd == 0) wnd = 1 << tc->rcv_wscale; } @@ -254,14 +253,12 @@ tcp_options_write (u8 * data, tcp_options_t * opts) if (tcp_opts_sack (opts)) { int i; - u32 n_sack_blocks = clib_min (vec_len (opts->sacks), - TCP_OPTS_MAX_SACK_BLOCKS); - if (n_sack_blocks != 0) + if (opts->n_sack_blocks != 0) { *data++ = TCP_OPTION_SACK_BLOCK; - *data++ = 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; - for (i = 0; i < n_sack_blocks; i++) + *data++ = 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; + for (i = 0; i < opts->n_sack_blocks; i++) { buf = clib_host_to_net_u32 (opts->sacks[i].start); clib_memcpy_fast (data, &buf, seq_len); @@ -270,7 +267,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts) clib_memcpy_fast (data, &buf, seq_len); data += seq_len; } - opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; + opts_len += 2 + opts->n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; } } @@ -373,9 +370,13 @@ tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts) if (vec_len (tc->snd_sacks)) { opts->flags |= TCP_OPTS_FLAG_SACK; - opts->sacks = tc->snd_sacks; - opts->n_sack_blocks = clib_min (vec_len (tc->snd_sacks), + if (tc->snd_sack_pos >= vec_len (tc->snd_sacks)) + tc->snd_sack_pos = 0; + opts->sacks = &tc->snd_sacks[tc->snd_sack_pos]; + opts->n_sack_blocks = vec_len (tc->snd_sacks) - tc->snd_sack_pos; + opts->n_sack_blocks = clib_min (opts->n_sack_blocks, TCP_OPTS_MAX_SACK_BLOCKS); + tc->snd_sack_pos += opts->n_sack_blocks; len += 2 + TCP_OPTION_LEN_SACK_BLOCK * opts->n_sack_blocks; } } @@ -463,41 +464,7 @@ tcp_init_mss (tcp_connection_t * tc) if (tcp_opts_tstamp (&tc->rcv_opts)) tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP; } - -static int -tcp_alloc_tx_buffers (tcp_worker_ctx_t * wrk, u16 * n_bufs, u32 wanted) -{ - vlib_main_t *vm = vlib_get_main (); - u32 n_alloc; - - ASSERT (wanted > *n_bufs); - vec_validate_aligned (wrk->tx_buffers, wanted - 1, CLIB_CACHE_LINE_BYTES); - n_alloc = vlib_buffer_alloc (vm, &wrk->tx_buffers[*n_bufs], - wanted - *n_bufs); - *n_bufs += n_alloc; - _vec_len (wrk->tx_buffers) = *n_bufs; - return n_alloc; -} - -always_inline int -tcp_get_free_buffer_index (tcp_worker_ctx_t * wrk, u32 * bidx) -{ - u16 n_bufs = vec_len (wrk->tx_buffers); - - TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (wrk->vm->thread_index); - - if (PREDICT_FALSE (!n_bufs)) - { - if (!tcp_alloc_tx_buffers (wrk, &n_bufs, VLIB_FRAME_SIZE)) - { - *bidx = ~0; - return -1; - } - } - *bidx = wrk->tx_buffers[--n_bufs]; - _vec_len (wrk->tx_buffers) = n_bufs; - return 0; -} +#endif /* CLIB_MARCH_VARIANT */ static void * tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b) @@ -512,27 +479,27 @@ tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b) vnet_buffer (b)->tcp.flags = 0; /* Leave enough space for headers */ - return vlib_buffer_make_headroom (b, MAX_HDRS_LEN); + return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN); } +#ifndef CLIB_MARCH_VARIANT static void * tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b) { ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - b->flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST; b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; b->total_length_not_including_first_buffer = 0; b->current_data = 0; vnet_buffer (b)->tcp.flags = 0; VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b); /* Leave enough space for headers */ - return vlib_buffer_make_headroom (b, MAX_HDRS_LEN); + return vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN); } /** * Prepare ACK */ -static void +static inline void tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state, u8 flags) { @@ -557,12 +524,9 @@ tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state, /** * Convert buffer to ACK */ -void +static inline void tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b) { - vlib_main_t *vm = vlib_get_main (); - - tcp_reuse_buffer (vm, b); tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK); TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc); tc->rcv_las = tc->rcv_nxt; @@ -574,13 +538,7 @@ tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b) void tcp_make_fin (tcp_connection_t * tc, vlib_buffer_t * b) { - vlib_main_t *vm = vlib_get_main (); - u8 flags = 0; - - tcp_reuse_buffer (vm, b); - - flags = TCP_FLAG_FIN | TCP_FLAG_ACK; - tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, flags); + tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_FIN | TCP_FLAG_ACK); /* Reset flags, make sure ack is sent */ vnet_buffer (b)->tcp.flags &= ~TCP_BUF_FLAG_DUPACK; @@ -617,15 +575,12 @@ tcp_make_syn (tcp_connection_t * tc, vlib_buffer_t * b) void tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b) { - vlib_main_t *vm = vlib_get_main (); tcp_options_t _snd_opts, *snd_opts = &_snd_opts; u8 tcp_opts_len, tcp_hdr_opts_len; tcp_header_t *th; u16 initial_wnd; clib_memset (snd_opts, 0, sizeof (*snd_opts)); - tcp_reuse_buffer (vm, b); - initial_wnd = tcp_initial_window_to_advertise (tc); tcp_opts_len = tcp_make_synack_options (tc, snd_opts); tcp_hdr_opts_len = tcp_opts_len + sizeof (tcp_header_t); @@ -636,11 +591,6 @@ tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b) tcp_options_write ((u8 *) (th + 1), snd_opts); vnet_buffer (b)->tcp.connection_index = tc->c_c_index; - - /* Init retransmit timer. Use update instead of set because of - * retransmissions */ - tcp_retransmit_timer_force_update (tc); - TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc); } always_inline void @@ -740,6 +690,7 @@ tcp_enqueue_to_output_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi, { tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1); } +#endif /* CLIB_MARCH_VARIANT */ static int tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, @@ -820,15 +771,17 @@ tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, return 0; } +#ifndef CLIB_MARCH_VARIANT /** * Send reset without reusing existing buffer * * It extracts connection info out of original packet */ void -tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4) +tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, + u32 thread_index, u8 is_ip4) { - tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + tcp_worker_ctx_t *wrk = tcp_get_worker (thread_index); vlib_main_t *vm = wrk->vm; vlib_buffer_t *b; u32 bi, sw_if_index, fib_index; @@ -839,7 +792,7 @@ tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4) ip6_header_t *ih6, *pkt_ih6; fib_protocol_t fib_proto; - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) return; b = vlib_get_buffer (vm, bi); @@ -915,7 +868,7 @@ tcp_send_reset (tcp_connection_t * tc) u16 tcp_hdr_opts_len, advertise_wnd, opts_write_len; u8 flags; - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) return; b = vlib_get_buffer (vm, bi); tcp_init_buffer (vm, b); @@ -994,12 +947,15 @@ tcp_send_syn (tcp_connection_t * tc) * Setup retransmit and establish timers before requesting buffer * such that we can return if we've ran out. */ - tcp_timer_set (tc, TCP_TIMER_ESTABLISH, TCP_ESTABLISH_TIME); + tcp_timer_set (tc, TCP_TIMER_ESTABLISH_AO, TCP_ESTABLISH_TIME); tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, tc->rto * TCP_TO_TIMER_TICK); - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) - return; + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) + { + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1); + return; + } b = vlib_get_buffer (vm, bi); tcp_init_buffer (vm, b); @@ -1023,14 +979,20 @@ tcp_send_synack (tcp_connection_t * tc) vlib_buffer_t *b; u32 bi; - /* Get buffer */ - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) - return; + tcp_retransmit_timer_force_update (tc); + + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) + { + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); + return; + } tc->rtt_ts = tcp_time_now_us (tc->c_thread_index); b = vlib_get_buffer (vm, bi); + tcp_init_buffer (vm, b); tcp_make_synack (tc, b); tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); + TCP_EVT_DBG (TCP_EVT_SYNACK_SENT, tc); } /** @@ -1092,11 +1054,16 @@ tcp_send_fin (tcp_connection_t * tc) if (fin_snt) tc->snd_nxt = tc->snd_una; - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { /* Out of buffers so program fin retransmit ASAP */ tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); - goto post_enqueue; + if (fin_snt) + tc->snd_nxt = tc->snd_una_max; + else + /* Make sure retransmit retries a fin not data */ + tc->flags |= TCP_CONN_FINSNT; + return; } tcp_retransmit_timer_force_update (tc); @@ -1106,7 +1073,6 @@ tcp_send_fin (tcp_connection_t * tc) tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4); TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc); -post_enqueue: if (!fin_snt) { tc->flags |= TCP_CONN_FINSNT; @@ -1130,18 +1096,14 @@ tcp_make_state_flags (tcp_connection_t * tc, tcp_state_t next_state) case TCP_STATE_CLOSE_WAIT: case TCP_STATE_TIME_WAIT: case TCP_STATE_FIN_WAIT_2: + case TCP_STATE_CLOSING: + case TCP_STATE_LAST_ACK: + case TCP_STATE_FIN_WAIT_1: return TCP_FLAG_ACK; case TCP_STATE_SYN_RCVD: return TCP_FLAG_SYN | TCP_FLAG_ACK; case TCP_STATE_SYN_SENT: return TCP_FLAG_SYN; - case TCP_STATE_LAST_ACK: - case TCP_STATE_FIN_WAIT_1: - case TCP_STATE_CLOSING: - if (tc->snd_nxt + 1 < tc->snd_una_max) - return TCP_FLAG_ACK; - else - return TCP_FLAG_FIN; default: clib_warning ("Shouldn't be here!"); } @@ -1211,8 +1173,9 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, } u32 -tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b) +tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) { + tcp_connection_t *tc = (tcp_connection_t *) tconn; tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, /* compute opts */ 0, /* burst */ 1); tc->snd_una_max = tc->snd_nxt; @@ -1241,13 +1204,13 @@ tcp_send_ack (tcp_connection_t * tc) vlib_buffer_t *b; u32 bi; - /* Get buffer */ - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) - return; + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) + { + tcp_update_rcv_wnd (tc); + return; + } b = vlib_get_buffer (vm, bi); tcp_init_buffer (vm, b); - - /* Fill in the ACK */ tcp_make_ack (tc, b); tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); } @@ -1290,14 +1253,34 @@ tcp_send_acks (tcp_worker_ctx_t * wrk) { tc = tcp_connection_get (pending_acks[i], thread_index); tc->flags &= ~TCP_CONN_SNDACK; - n_acks = clib_max (1, tc->pending_dupacks); + if (!tc->pending_dupacks) + { + tcp_send_ack (tc); + continue; + } + /* If we're supposed to send dupacks but have no ooo data * send only one ack */ - if (tc->pending_dupacks && !vec_len (tc->snd_sacks)) - n_acks = 1; + if (!vec_len (tc->snd_sacks)) + { + tcp_send_ack (tc); + continue; + } + + /* Start with first sack block */ + tc->snd_sack_pos = 0; + + /* Generate enough dupacks to cover all sack blocks. Do not generate + * more sacks than the number of packets received. But do generate at + * least 3, i.e., the number needed to signal congestion, if needed. */ + n_acks = vec_len (tc->snd_sacks) / TCP_OPTS_MAX_SACK_BLOCKS; + n_acks = clib_min (n_acks, tc->pending_dupacks); + n_acks = clib_max (n_acks, clib_min (tc->pending_dupacks, 3)); for (j = 0; j < n_acks; j++) tcp_send_ack (tc); + tc->pending_dupacks = 0; + tc->snd_sack_pos = 0; } _vec_len (wrk->pending_acks) = 0; } @@ -1335,12 +1318,12 @@ tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b) { u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer; - u32 bi, seg_size; vlib_main_t *vm = wrk->vm; + u32 bi, seg_size; int n_bytes = 0; u8 *data; - seg_size = max_deq_bytes + MAX_HDRS_LEN; + seg_size = max_deq_bytes + TRANSPORT_MAX_HDRS_LEN; /* * Prepare options @@ -1354,12 +1337,12 @@ tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, /* Easy case, buffer size greater than mss */ if (PREDICT_TRUE (seg_size <= bytes_per_buffer)) { - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) return 0; *b = vlib_get_buffer (vm, bi); data = tcp_init_buffer (vm, *b); - n_bytes = stream_session_peek_bytes (&tc->connection, data, offset, - max_deq_bytes); + n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset, + max_deq_bytes); ASSERT (n_bytes == max_deq_bytes); b[0]->current_length = n_bytes; tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0); @@ -1369,30 +1352,28 @@ tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, /* Split mss into multiple buffers */ else { - u32 chain_bi = ~0, n_bufs_per_seg; - u16 n_peeked, len_to_deq, available_bufs; + u32 chain_bi = ~0, n_bufs_per_seg, n_bufs; + u16 n_peeked, len_to_deq; vlib_buffer_t *chain_b, *prev_b; int i; /* Make sure we have enough buffers */ n_bufs_per_seg = ceil ((double) seg_size / bytes_per_buffer); - available_bufs = vec_len (wrk->tx_buffers); - if (n_bufs_per_seg > available_bufs) + vec_validate_aligned (wrk->tx_buffers, n_bufs_per_seg - 1, + CLIB_CACHE_LINE_BYTES); + n_bufs = vlib_buffer_alloc (vm, wrk->tx_buffers, n_bufs_per_seg); + if (PREDICT_FALSE (n_bufs != n_bufs_per_seg)) { - tcp_alloc_tx_buffers (wrk, &available_bufs, VLIB_FRAME_SIZE); - if (n_bufs_per_seg > available_bufs) - { - *b = 0; - return 0; - } + if (n_bufs) + vlib_buffer_free (vm, wrk->tx_buffers, n_bufs); + return 0; } - (void) tcp_get_free_buffer_index (wrk, &bi); - ASSERT (bi != (u32) ~ 0); - *b = vlib_get_buffer (vm, bi); + *b = vlib_get_buffer (vm, wrk->tx_buffers[--n_bufs]); data = tcp_init_buffer (vm, *b); - n_bytes = stream_session_peek_bytes (&tc->connection, data, offset, - bytes_per_buffer - MAX_HDRS_LEN); + n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset, + bytes_per_buffer - + TRANSPORT_MAX_HDRS_LEN); b[0]->current_length = n_bytes; b[0]->flags |= VLIB_BUFFER_TOTAL_LENGTH_VALID; b[0]->total_length_not_including_first_buffer = 0; @@ -1403,13 +1384,13 @@ tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, { prev_b = chain_b; len_to_deq = clib_min (max_deq_bytes, bytes_per_buffer); - tcp_get_free_buffer_index (wrk, &chain_bi); - ASSERT (chain_bi != (u32) ~ 0); + chain_bi = wrk->tx_buffers[--n_bufs]; chain_b = vlib_get_buffer (vm, chain_bi); chain_b->current_data = 0; data = vlib_buffer_get_current (chain_b); - n_peeked = stream_session_peek_bytes (&tc->connection, data, - offset + n_bytes, len_to_deq); + n_peeked = session_tx_fifo_peek_bytes (&tc->connection, data, + offset + n_bytes, + len_to_deq); ASSERT (n_peeked == len_to_deq); n_bytes += n_peeked; chain_b->current_length = n_peeked; @@ -1426,6 +1407,12 @@ tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0); if (seq_gt (tc->snd_nxt, tc->snd_una_max)) tc->snd_una_max = tc->snd_nxt; + + if (PREDICT_FALSE (n_bufs)) + { + clib_warning ("not all buffers consumed"); + vlib_buffer_free (vm, wrk->tx_buffers, n_bufs); + } } ASSERT (n_bytes > 0); @@ -1454,7 +1441,7 @@ tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk, /* * Make sure we can retransmit something */ - available_bytes = session_tx_fifo_max_dequeue (&tc->connection); + available_bytes = transport_max_tx_dequeue (&tc->connection); ASSERT (available_bytes >= offset); available_bytes -= offset; if (!available_bytes) @@ -1505,6 +1492,8 @@ tcp_rxt_timeout_cc (tcp_connection_t * tc) scoreboard_clear (&tc->sack_sb); tcp_cc_fastrecovery_exit (tc); } + else + tc->rcv_dupacks = 0; /* Start again from the beginning */ tc->cc_algo->congestion (tc); @@ -1538,9 +1527,12 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { tc = tcp_connection_get (index, thread_index); /* Note: the connection may have been closed and pool_put */ - if (PREDICT_FALSE (tc == 0 || tc->state < TCP_STATE_SYN_RCVD)) + if (PREDICT_FALSE (tc == 0 || tc->state == TCP_STATE_SYN_SENT)) return; tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID; + /* Wait-close and retransmit could pop at the same time */ + if (tc->state == TCP_STATE_CLOSED) + return; } if (tc->state >= TCP_STATE_ESTABLISHED) @@ -1548,7 +1540,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 2); /* Lost FIN, retransmit and return */ - if (tcp_is_lost_fin (tc)) + if (tc->flags & TCP_CONN_FINSNT) { tcp_send_fin (tc); tc->rto_boff += 1; @@ -1595,10 +1587,9 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) /* Send one segment. Note that n_bytes may be zero due to buffer * shortfall */ n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b); - - if (n_bytes == 0) + if (!n_bytes) { - tcp_retransmit_timer_force_update (tc); + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); return; } @@ -1620,10 +1611,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) if (tc->flags & TCP_CONN_HALF_OPEN_DONE) { if (tcp_half_open_connection_cleanup (tc)) - { - clib_warning ("could not remove half-open connection"); - ASSERT (0); - } + TCP_DBG ("could not remove half-open connection"); return; } @@ -1638,8 +1626,11 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, tc->rto * TCP_TO_TIMER_TICK); - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) - return; + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) + { + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, 1); + return; + } b = vlib_get_buffer (vm, bi); tcp_init_buffer (vm, b); @@ -1662,9 +1653,11 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); tc->rtt_ts = 0; - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + tcp_retransmit_timer_force_update (tc); + + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { - tcp_retransmit_timer_force_update (tc); + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1); return; } @@ -1713,7 +1706,6 @@ tcp_timer_persist_handler (u32 index) u8 *data; tc = tcp_connection_get_if_valid (index, thread_index); - if (!tc) return; @@ -1721,11 +1713,11 @@ tcp_timer_persist_handler (u32 index) tc->timers[TCP_TIMER_PERSIST] = TCP_TIMER_HANDLE_INVALID; /* Problem already solved or worse */ - if (tc->state == TCP_STATE_CLOSED || tc->state > TCP_STATE_ESTABLISHED - || tc->snd_wnd > tc->snd_mss) + if (tc->state == TCP_STATE_CLOSED || tc->snd_wnd > tc->snd_mss + || (tc->flags & TCP_CONN_FINSNT)) return; - available_bytes = session_tx_fifo_max_dequeue (&tc->connection); + available_bytes = transport_max_tx_dequeue (&tc->connection); offset = tc->snd_una_max - tc->snd_una; /* Reprogram persist if no new bytes available to send. We may have data @@ -1749,7 +1741,7 @@ tcp_timer_persist_handler (u32 index) /* * Try to force the first unsent segment (or buffer) */ - if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { tcp_persist_timer_set (tc); return; @@ -1759,9 +1751,10 @@ tcp_timer_persist_handler (u32 index) tcp_validate_txf_size (tc, offset); tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state); - max_snd_bytes = clib_min (tc->snd_mss, tm->bytes_per_buffer - MAX_HDRS_LEN); - n_bytes = stream_session_peek_bytes (&tc->connection, data, offset, - max_snd_bytes); + max_snd_bytes = clib_min (tc->snd_mss, + tm->bytes_per_buffer - TRANSPORT_MAX_HDRS_LEN); + n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset, + max_snd_bytes); b->current_length = n_bytes; ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT) || tc->snd_nxt == tc->snd_una_max @@ -1863,7 +1856,7 @@ tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, sb = &tc->sack_sb; hole = scoreboard_get_hole (sb, sb->cur_rxt_hole); - max_deq = session_tx_fifo_max_dequeue (&tc->connection); + max_deq = transport_max_tx_dequeue (&tc->connection); max_deq -= tc->snd_una_max - tc->snd_una; while (snd_space > 0 && n_segs < burst_size) @@ -1990,7 +1983,7 @@ send_unsent: if (snd_space < tc->snd_mss || tc->snd_mss == 0) goto done; - max_deq = session_tx_fifo_max_dequeue (&tc->connection); + max_deq = transport_max_tx_dequeue (&tc->connection); max_deq -= tc->snd_una_max - tc->snd_una; if (max_deq) { @@ -2022,6 +2015,7 @@ tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, else return tcp_fast_retransmit_no_sack (wrk, tc, burst_size); } +#endif /* CLIB_MARCH_VARIANT */ static void tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0, @@ -2207,16 +2201,14 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, return frame->n_vectors; } -static uword -tcp4_output (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) +VLIB_NODE_FN (tcp4_output_node) (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) { return tcp46_output_inline (vm, node, from_frame, 1 /* is_ip4 */ ); } -static uword -tcp6_output (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) +VLIB_NODE_FN (tcp6_output_node) (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) { return tcp46_output_inline (vm, node, from_frame, 0 /* is_ip4 */ ); } @@ -2224,7 +2216,6 @@ tcp6_output (vlib_main_t * vm, vlib_node_runtime_t * node, /* *INDENT-OFF* */ VLIB_REGISTER_NODE (tcp4_output_node) = { - .function = tcp4_output, .name = "tcp4-output", /* Takes a vector of packets. */ .vector_size = sizeof (u32), @@ -2242,12 +2233,9 @@ VLIB_REGISTER_NODE (tcp4_output_node) = }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (tcp4_output_node, tcp4_output); - /* *INDENT-OFF* */ VLIB_REGISTER_NODE (tcp6_output_node) = { - .function = tcp6_output, .name = "tcp6-output", /* Takes a vector of packets. */ .vector_size = sizeof (u32), @@ -2265,8 +2253,6 @@ VLIB_REGISTER_NODE (tcp6_output_node) = }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output); - typedef enum _tcp_reset_next { TCP_RESET_NEXT_DROP, @@ -2352,23 +2338,20 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, return from_frame->n_vectors; } -static uword -tcp4_send_reset (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) +VLIB_NODE_FN (tcp4_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) { return tcp46_send_reset_inline (vm, node, from_frame, 1); } -static uword -tcp6_send_reset (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame) +VLIB_NODE_FN (tcp6_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node, + vlib_frame_t * from_frame) { return tcp46_send_reset_inline (vm, node, from_frame, 0); } /* *INDENT-OFF* */ VLIB_REGISTER_NODE (tcp4_reset_node) = { - .function = tcp4_send_reset, .name = "tcp4-reset", .vector_size = sizeof (u32), .n_errors = TCP_N_ERROR, @@ -2383,11 +2366,8 @@ VLIB_REGISTER_NODE (tcp4_reset_node) = { }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (tcp4_reset_node, tcp4_send_reset); - /* *INDENT-OFF* */ VLIB_REGISTER_NODE (tcp6_reset_node) = { - .function = tcp6_send_reset, .name = "tcp6-reset", .vector_size = sizeof (u32), .n_errors = TCP_N_ERROR, @@ -2402,8 +2382,6 @@ VLIB_REGISTER_NODE (tcp6_reset_node) = { }; /* *INDENT-ON* */ -VLIB_NODE_FUNCTION_MULTIARCH (tcp6_reset_node, tcp6_send_reset); - /* * fd.io coding-style-patch-verification: ON *