X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp_output.c;h=8a88bf80b13b1d5b581c5c999f1071b1dfe12f33;hb=25579b4acd449e1bae30d2a20a44b77741c8e1fd;hp=cb1fcc9ab4987165f308997b99484adfac6a1072;hpb=f1762d6fa8cbc1a8fae691e568e73e94d5dcbc93;p=vpp.git diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index cb1fcc9ab49..8a88bf80b13 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -24,16 +24,22 @@ typedef enum _tcp_output_next { TCP_OUTPUT_NEXT_DROP, TCP_OUTPUT_NEXT_IP_LOOKUP, + TCP_OUTPUT_NEXT_IP_REWRITE, + TCP_OUTPUT_NEXT_IP_ARP, TCP_OUTPUT_N_NEXT } tcp_output_next_t; #define foreach_tcp4_output_next \ _ (DROP, "error-drop") \ - _ (IP_LOOKUP, "ip4-lookup") + _ (IP_LOOKUP, "ip4-lookup") \ + _ (IP_REWRITE, "ip4-rewrite") \ + _ (IP_ARP, "ip4-arp") #define foreach_tcp6_output_next \ _ (DROP, "error-drop") \ - _ (IP_LOOKUP, "ip6-lookup") + _ (IP_LOOKUP, "ip6-lookup") \ + _ (IP_REWRITE, "ip6-rewrite") \ + _ (IP_ARP, "ip6-discover-neighbor") static char *tcp_error_strings[] = { #define tcp_error(n,s) s, @@ -55,7 +61,7 @@ format_tcp_tx_trace (u8 * s, va_list * args) CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *); CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *); tcp_tx_trace_t *t = va_arg (*args, tcp_tx_trace_t *); - uword indent = format_get_indent (s); + u32 indent = format_get_indent (s); s = format (s, "%U\n%U%U", format_tcp_header, &t->tcp_header, 128, @@ -157,8 +163,8 @@ tcp_update_rcv_wnd (tcp_connection_t * tc) /* * Figure out how much space we have available */ - available_space = stream_session_max_rx_enqueue (&tc->connection); - max_fifo = stream_session_rx_fifo_size (&tc->connection); + available_space = transport_max_rx_enqueue (&tc->connection); + max_fifo = transport_rx_fifo_size (&tc->connection); ASSERT (tc->rcv_opts.mss < max_fifo); if (available_space < tc->rcv_opts.mss && available_space < max_fifo >> 3) @@ -383,13 +389,15 @@ tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts, { case TCP_STATE_ESTABLISHED: case TCP_STATE_FIN_WAIT_1: + case TCP_STATE_CLOSED: + case TCP_STATE_CLOSE_WAIT: return tcp_make_established_options (tc, opts); case TCP_STATE_SYN_RCVD: return tcp_make_synack_options (tc, opts); case TCP_STATE_SYN_SENT: return tcp_make_syn_options (opts, tc->rcv_wscale); default: - clib_warning ("Not handled!"); + clib_warning ("State not handled! %d", state); return 0; } } @@ -437,49 +445,43 @@ tcp_init_mss (tcp_connection_t * tc) } always_inline int -tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u32 n_free_buffers) +tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs, + u32 wanted) { vlib_main_t *vm = vlib_get_main (); - u32 current_length = vec_len (tm->tx_buffers[thread_index]); - u32 n_allocated; - - vec_validate (tm->tx_buffers[thread_index], - current_length + n_free_buffers - 1); - n_allocated = - vlib_buffer_alloc (vm, &tm->tx_buffers[thread_index][current_length], - n_free_buffers); - _vec_len (tm->tx_buffers[thread_index]) = current_length + n_allocated; - /* buffer shortage, report failure */ - if (vec_len (tm->tx_buffers[thread_index]) == 0) - { - clib_warning ("out of buffers"); - return -1; - } - return 0; + u32 n_alloc; + + ASSERT (wanted > *n_bufs); + vec_validate_aligned (tm->tx_buffers[thread_index], wanted - 1, + CLIB_CACHE_LINE_BYTES); + n_alloc = vlib_buffer_alloc (vm, &tm->tx_buffers[thread_index][*n_bufs], + wanted - *n_bufs); + *n_bufs += n_alloc; + _vec_len (tm->tx_buffers[thread_index]) = *n_bufs; + return n_alloc; } always_inline int tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx) { - u32 *my_tx_buffers; u32 thread_index = vlib_get_thread_index (); - if (PREDICT_FALSE (vec_len (tm->tx_buffers[thread_index]) == 0)) + u16 n_bufs = vec_len (tm->tx_buffers[thread_index]); + + TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (thread_index); + + if (PREDICT_FALSE (!n_bufs)) { - if (tcp_alloc_tx_buffers (tm, thread_index, VLIB_FRAME_SIZE)) - return -1; + if (!tcp_alloc_tx_buffers (tm, thread_index, &n_bufs, VLIB_FRAME_SIZE)) + { + *bidx = ~0; + return -1; + } } - my_tx_buffers = tm->tx_buffers[thread_index]; - *bidx = my_tx_buffers[vec_len (my_tx_buffers) - 1]; - _vec_len (my_tx_buffers) -= 1; + *bidx = tm->tx_buffers[thread_index][--n_bufs]; + _vec_len (tm->tx_buffers[thread_index]) = n_bufs; return 0; } -always_inline void -tcp_return_buffer (tcp_main_t * tm) -{ - _vec_len (tm->tx_buffers[vlib_get_thread_index ()]) += 1; -} - always_inline void * tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b) { @@ -500,11 +502,12 @@ always_inline void * tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b) { ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0); - b->flags &= VLIB_BUFFER_FREE_LIST_INDEX_MASK; + b->flags &= VLIB_BUFFER_NON_DEFAULT_FREELIST; b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; b->total_length_not_including_first_buffer = 0; + b->current_data = 0; vnet_buffer (b)->tcp.flags = 0; - + VLIB_BUFFER_TRACE_TRAJECTORY_INIT (b); /* Leave enough space for headers */ return vlib_buffer_make_headroom (b, MAX_HDRS_LEN); } @@ -590,9 +593,6 @@ tcp_make_syn (tcp_connection_t * tc, vlib_buffer_t * b) initial_wnd); vnet_buffer (b)->tcp.connection_index = tc->c_c_index; tcp_options_write ((u8 *) (th + 1), &snd_opts); - - tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, - tc->rto * TCP_TO_TIMER_TICK); } /** @@ -630,7 +630,7 @@ tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b) always_inline void tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, - u8 is_ip4, u8 flush) + u8 is_ip4, u32 fib_index, u8 flush) { tcp_main_t *tm = vnet_get_tcp_main (); u32 thread_index = vlib_get_thread_index (); @@ -640,16 +640,12 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; b->error = 0; - /* Default FIB for now */ - vnet_buffer (b)->sw_if_index[VLIB_TX] = 0; + vnet_buffer (b)->sw_if_index[VLIB_TX] = fib_index; + vnet_buffer (b)->sw_if_index[VLIB_RX] = 0; /* Send to IP lookup */ next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index; - if (VLIB_BUFFER_TRACE_TRAJECTORY > 0) - { - b->pre_data[0] = 2; - b->pre_data[1] = next_index; - } + tcp_trajectory_add_start (b, 1); f = tm->ip_lookup_tx_frames[!is_ip4][thread_index]; if (!f) @@ -671,16 +667,18 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, always_inline void tcp_enqueue_to_ip_lookup_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, - u8 is_ip4) + u8 is_ip4, u32 fib_index) { - tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, 1); + tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 1); } always_inline void tcp_enqueue_to_ip_lookup (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, - u8 is_ip4) + u8 is_ip4, u32 fib_index) { - tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, 0); + tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 0); + if (vm->thread_index == 0 && vlib_num_workers ()) + session_flush_frames_main_thread (vm); } always_inline void @@ -697,11 +695,7 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, /* Decide where to send the packet */ next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index; - if (VLIB_BUFFER_TRACE_TRAJECTORY > 0) - { - b->pre_data[0] = 1; - b->pre_data[1] = next_index; - } + tcp_trajectory_add_start (b, 2); /* Get frame to v4/6 output node */ f = tm->tx_frames[!is_ip4][thread_index]; @@ -791,6 +785,7 @@ tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, } tcp_reuse_buffer (vm, b0); + tcp_trajectory_add_start (b0, 4); th0 = vlib_buffer_push_tcp_net_order (b0, dst_port, src_port, seq, ack, sizeof (tcp_header_t), flags, 0); @@ -821,7 +816,7 @@ void tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4) { vlib_buffer_t *b; - u32 bi; + u32 bi, sw_if_index, fib_index; tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); u8 tcp_hdr_len, flags = 0; @@ -829,11 +824,15 @@ tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4) u32 seq, ack; ip4_header_t *ih4, *pkt_ih4; ip6_header_t *ih6, *pkt_ih6; + fib_protocol_t fib_proto; if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) return; b = vlib_get_buffer (vm, bi); + sw_if_index = vnet_buffer (pkt)->sw_if_index[VLIB_RX]; + fib_proto = is_ip4 ? FIB_PROTOCOL_IP4 : FIB_PROTOCOL_IP6; + fib_index = fib_table_get_index_for_sw_if_index (fib_proto, sw_if_index); tcp_init_buffer (vm, b); /* Make and write options */ @@ -885,7 +884,7 @@ tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, u8 is_ip4) ASSERT (!bogus); } - tcp_enqueue_to_ip_lookup_now (vm, b, bi, is_ip4); + tcp_enqueue_to_ip_lookup_now (vm, b, bi, is_ip4, fib_index); TCP_EVT_DBG (TCP_EVT_RST_SENT, tc); } @@ -934,7 +933,7 @@ tcp_send_reset (tcp_connection_t * tc) th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus); ASSERT (!bogus); } - tcp_enqueue_to_ip_lookup_now (vm, b, bi, tc->c_is_ip4); + tcp_enqueue_to_ip_lookup_now (vm, b, bi, tc->c_is_ip4, tc->c_fib_index); TCP_EVT_DBG (TCP_EVT_RST_SENT, tc); } @@ -977,6 +976,14 @@ tcp_send_syn (tcp_connection_t * tc) tcp_main_t *tm = vnet_get_tcp_main (); vlib_main_t *vm = vlib_get_main (); + /* + * Setup retransmit and establish timers before requesting buffer + * such that we can return if we've ran out. + */ + tcp_timer_set (tc, TCP_TIMER_ESTABLISH, TCP_ESTABLISH_TIME); + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, + tc->rto * TCP_TO_TIMER_TICK); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) return; @@ -989,11 +996,8 @@ tcp_send_syn (tcp_connection_t * tc) tc->rtt_seq = tc->snd_nxt; tc->rto_boff = 0; - /* Set the connection establishment timer */ - tcp_timer_set (tc, TCP_TIMER_ESTABLISH, TCP_ESTABLISH_TIME); - tcp_push_ip_hdr (tm, tc, b); - tcp_enqueue_to_ip_lookup (vm, b, bi, tc->c_is_ip4); + tcp_enqueue_to_ip_lookup (vm, b, bi, tc->c_is_ip4, tc->c_fib_index); TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc); } @@ -1055,10 +1059,11 @@ tcp_send_fin (tcp_connection_t * tc) u32 bi; u8 fin_snt = 0; - + tcp_retransmit_timer_force_update (tc); if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) return; b = vlib_get_buffer (vm, bi); + tcp_init_buffer (vm, b); fin_snt = tc->flags & TCP_CONN_FINSNT; if (fin_snt) tc->snd_nxt = tc->snd_una; @@ -1072,7 +1077,10 @@ tcp_send_fin (tcp_connection_t * tc) tc->snd_una_max += 1; tc->snd_nxt = tc->snd_una_max; } - tcp_retransmit_timer_force_update (tc); + else + { + tc->snd_nxt = tc->snd_una_max; + } TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc); } @@ -1161,6 +1169,7 @@ tcp_send_ack (tcp_connection_t * tc) if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) return; b = vlib_get_buffer (vm, bi); + tcp_init_buffer (vm, b); /* Fill in the ACK */ tcp_make_ack (tc, b); @@ -1205,7 +1214,7 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, /* * Make sure we can retransmit something */ - available_bytes = stream_session_tx_fifo_max_dequeue (&tc->connection); + available_bytes = session_tx_fifo_max_dequeue (&tc->connection); ASSERT (available_bytes >= offset); available_bytes -= offset; if (!available_bytes) @@ -1237,14 +1246,13 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, * Allocate and fill in buffer(s) */ - if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) - return 0; - *b = vlib_get_buffer (vm, bi); - data = tcp_init_buffer (vm, *b); - /* Easy case, buffer size greater than mss */ if (PREDICT_TRUE (seg_size <= tm->bytes_per_buffer)) { + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) + return 0; + *b = vlib_get_buffer (vm, bi); + data = tcp_init_buffer (vm, *b); n_bytes = stream_session_peek_bytes (&tc->connection, data, offset, max_deq_bytes); ASSERT (n_bytes == max_deq_bytes); @@ -1260,21 +1268,25 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, vlib_buffer_t *chain_b, *prev_b; int i; - n_bufs_per_seg = ceil ((double) seg_size / tm->bytes_per_buffer); - /* Make sure we have enough buffers */ + n_bufs_per_seg = ceil ((double) seg_size / tm->bytes_per_buffer); available_bufs = vec_len (tm->tx_buffers[thread_index]); if (n_bufs_per_seg > available_bufs) { - if (tcp_alloc_tx_buffers (tm, thread_index, - VLIB_FRAME_SIZE - available_bufs)) + tcp_alloc_tx_buffers (tm, thread_index, &available_bufs, + VLIB_FRAME_SIZE); + + if (n_bufs_per_seg > available_bufs) { - tcp_return_buffer (tm); *b = 0; return 0; } } + tcp_get_free_buffer_index (tm, &bi); + ASSERT (bi != (u32) ~ 0); + *b = vlib_get_buffer (vm, bi); + data = tcp_init_buffer (vm, *b); n_bytes = stream_session_peek_bytes (&tc->connection, data, offset, tm->bytes_per_buffer - MAX_HDRS_LEN); @@ -1298,7 +1310,6 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset, ASSERT (n_peeked == len_to_deq); n_bytes += n_peeked; chain_b->current_length = n_peeked; - chain_b->flags &= VLIB_BUFFER_FREE_LIST_INDEX_MASK; chain_b->next_buffer = 0; /* update previous buffer */ @@ -1328,8 +1339,9 @@ done: * Reset congestion control, switch cwnd to loss window and try again. */ static void -tcp_rtx_timeout_cc (tcp_connection_t * tc) +tcp_rxt_timeout_cc (tcp_connection_t * tc) { + TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 6); tc->prev_ssthresh = tc->ssthresh; tc->prev_cwnd = tc->cwnd; @@ -1338,10 +1350,12 @@ tcp_rtx_timeout_cc (tcp_connection_t * tc) tcp_cc_fastrecovery_exit (tc); /* Start again from the beginning */ - tc->ssthresh = clib_max (tcp_flight_size (tc) / 2, 2 * tc->snd_mss); + tc->cc_algo->congestion (tc); tc->cwnd = tcp_loss_wnd (tc); tc->snd_congestion = tc->snd_una_max; tc->rtt_ts = 0; + tc->cwnd_acc_bytes = 0; + tcp_recovery_on (tc); } @@ -1372,12 +1386,24 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID; } + TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1); + if (tc->state >= TCP_STATE_ESTABLISHED) { /* Lost FIN, retransmit and return */ if (tcp_is_lost_fin (tc)) { tcp_send_fin (tc); + tc->rto_boff += 1; + tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); + return; + } + + /* Shouldn't be here */ + if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_una_max) + || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion))) + { + tcp_recovery_off (tc); return; } @@ -1391,17 +1417,14 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) /* Increment RTO backoff (also equal to number of retries) and go back * to first un-acked byte */ tc->rto_boff += 1; - tc->snd_nxt = tc->snd_una; /* First retransmit timeout */ if (tc->rto_boff == 1) - tcp_rtx_timeout_cc (tc); + tcp_rxt_timeout_cc (tc); - /* Exponential backoff */ + tc->snd_una_max = tc->snd_nxt = tc->snd_una; tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); - TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1); - /* Send one segment. Note that n_bytes may be zero due to buffer shortfall */ n_bytes = tcp_prepare_retransmit_segment (tc, 0, tc->snd_mss, &b); @@ -1410,12 +1433,6 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) if (n_bytes == 0) { - ASSERT (!b); - if (tc->snd_una == tc->snd_una_max) - return; - ASSERT (tc->rto_boff > 1 && tc->snd_una == tc->snd_congestion); - clib_warning ("retransmit fail: %U", format_tcp_connection, tc, 2); - /* Try again eventually */ tcp_retransmit_timer_set (tc); return; } @@ -1451,6 +1468,9 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) if (tc->rto_boff > TCP_RTO_SYN_RETRIES) tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX); + tcp_timer_update (tc, TCP_TIMER_RETRANSMIT_SYN, + tc->rto * TCP_TO_TIMER_TICK); + if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) return; @@ -1463,7 +1483,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) /* This goes straight to ipx_lookup. Retransmit timer set already */ tcp_push_ip_hdr (tm, tc, b); - tcp_enqueue_to_ip_lookup (vm, b, bi, tc->c_is_ip4); + tcp_enqueue_to_ip_lookup (vm, b, bi, tc->c_is_ip4, tc->c_fib_index); } /* Retransmit SYN-ACK */ else if (tc->state == TCP_STATE_SYN_RCVD) @@ -1474,9 +1494,13 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) tc->rtt_ts = 0; if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi))) - return; + { + tcp_retransmit_timer_force_update (tc); + return; + } b = vlib_get_buffer (vm, bi); + tcp_init_buffer (vm, b); tcp_make_synack (tc, b); TCP_EVT_DBG (TCP_EVT_SYN_RXT, tc, 1); @@ -1486,7 +1510,6 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) else { ASSERT (tc->state == TCP_STATE_CLOSED); - TCP_DBG ("connection state: %d", tc->state); return; } } @@ -1515,7 +1538,7 @@ tcp_timer_persist_handler (u32 index) u32 thread_index = vlib_get_thread_index (); tcp_connection_t *tc; vlib_buffer_t *b; - u32 bi, old_snd_nxt, max_snd_bytes, available_bytes, offset; + u32 bi, max_snd_bytes, available_bytes, offset; int n_bytes = 0; u8 *data; @@ -1532,7 +1555,7 @@ tcp_timer_persist_handler (u32 index) || tc->snd_wnd > tc->snd_mss || tcp_in_recovery (tc)) return; - available_bytes = stream_session_tx_fifo_max_dequeue (&tc->connection); + available_bytes = session_tx_fifo_max_dequeue (&tc->connection); offset = tc->snd_una_max - tc->snd_una; /* Reprogram persist if no new bytes available to send. We may have data @@ -1567,14 +1590,11 @@ tcp_timer_persist_handler (u32 index) n_bytes = stream_session_peek_bytes (&tc->connection, data, offset, max_snd_bytes); b->current_length = n_bytes; - ASSERT (n_bytes != 0 && (tc->snd_nxt == tc->snd_una_max || tc->rto_boff > 1 - || tcp_timer_is_active (tc, - TCP_TIMER_RETRANSMIT))); + ASSERT (n_bytes != 0 && (tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT) + || tc->snd_nxt == tc->snd_una_max + || tc->rto_boff > 1)); - /* Allow updating of snd_una_max but don't update snd_nxt */ - old_snd_nxt = tc->snd_nxt; tcp_push_hdr_i (tc, b, tc->state, 0); - tc->snd_nxt = old_snd_nxt; tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4); /* Just sent new data, enable retransmit */ @@ -1611,7 +1631,7 @@ void tcp_fast_retransmit_sack (tcp_connection_t * tc) { vlib_main_t *vm = vlib_get_main (); - u32 n_written = 0, offset, max_bytes; + u32 n_written = 0, offset, max_bytes, n_segs = 0; vlib_buffer_t *b = 0; sack_scoreboard_hole_t *hole; sack_scoreboard_t *sb; @@ -1620,14 +1640,17 @@ tcp_fast_retransmit_sack (tcp_connection_t * tc) u8 snd_limited = 0, can_rescue = 0; ASSERT (tcp_in_fastrecovery (tc)); - TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0); old_snd_nxt = tc->snd_nxt; sb = &tc->sack_sb; - snd_space = tcp_available_snd_space (tc); + snd_space = tcp_available_cc_snd_space (tc); + if (snd_space < tc->snd_mss) + goto done; + + TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0); hole = scoreboard_get_hole (sb, sb->cur_rxt_hole); - while (hole && snd_space > 0) + while (hole && snd_space > 0 && n_segs++ < VLIB_FRAME_SIZE) { hole = scoreboard_next_rxt_hole (sb, hole, tcp_fastrecovery_sent_1_smss (tc), @@ -1652,7 +1675,9 @@ tcp_fast_retransmit_sack (tcp_connection_t * tc) tc->snd_nxt = tc->snd_una + offset; n_written = tcp_prepare_retransmit_segment (tc, offset, max_bytes, &b); - ASSERT (n_written); + if (!n_written) + goto done; + bi = vlib_get_buffer_index (vm, b); tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4); break; @@ -1677,6 +1702,7 @@ tcp_fast_retransmit_sack (tcp_connection_t * tc) snd_space -= n_written; } +done: /* If window allows, send 1 SMSS of new data */ tc->snd_nxt = old_snd_nxt; } @@ -1698,7 +1724,7 @@ tcp_fast_retransmit_no_sack (tcp_connection_t * tc) /* Start resending from first un-acked segment */ old_snd_nxt = tc->snd_nxt; tc->snd_nxt = tc->snd_una; - snd_space = tcp_available_snd_space (tc); + snd_space = tcp_available_cc_snd_space (tc); while (snd_space > 0) { @@ -1724,8 +1750,7 @@ tcp_fast_retransmit_no_sack (tcp_connection_t * tc) void tcp_fast_retransmit (tcp_connection_t * tc) { - if (tcp_opts_sack_permitted (&tc->rcv_opts) - && scoreboard_first_hole (&tc->sack_sb)) + if (tcp_opts_sack_permitted (&tc->rcv_opts)) tcp_fast_retransmit_sack (tc); else tcp_fast_retransmit_no_sack (tc); @@ -1734,11 +1759,42 @@ tcp_fast_retransmit (tcp_connection_t * tc) always_inline u32 tcp_session_has_ooo_data (tcp_connection_t * tc) { - stream_session_t *s = - stream_session_get (tc->c_s_index, tc->c_thread_index); + stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index); return svm_fifo_has_ooo_data (s->server_rx_fifo); } +static void +tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0, + u32 * next0, u32 * error0) +{ + ip_adjacency_t *adj; + adj_index_t ai; + + /* Not thread safe but as long as the connection exists the adj should + * not be removed */ + ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip, + tc0->sw_if_index); + if (ai == ADJ_INDEX_INVALID) + { + vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0; + *next0 = TCP_OUTPUT_NEXT_DROP; + *error0 = TCP_ERROR_LINK_LOCAL_RW; + return; + } + + adj = adj_get (ai); + if (PREDICT_TRUE (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)) + *next0 = TCP_OUTPUT_NEXT_IP_REWRITE; + else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP) + *next0 = TCP_OUTPUT_NEXT_IP_ARP; + else + { + *next0 = TCP_OUTPUT_NEXT_DROP; + *error0 = TCP_ERROR_LINK_LOCAL_RW; + } + vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai; +} + always_inline uword tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, @@ -1767,6 +1823,14 @@ tcp46_output_inline (vlib_main_t * vm, tcp_header_t *th0 = 0; u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_IP_LOOKUP; + if (n_left_from > 1) + { + vlib_buffer_t *pb; + pb = vlib_get_buffer (vm, from[1]); + vlib_prefetch_buffer_header (pb, STORE); + CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, STORE); + } + bi0 = from[0]; to_next[0] = bi0; from += 1; @@ -1786,6 +1850,8 @@ tcp46_output_inline (vlib_main_t * vm, th0 = vlib_buffer_get_current (b0); TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length); + vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index; + vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0; if (is_ip4) { @@ -1804,12 +1870,22 @@ tcp46_output_inline (vlib_main_t * vm, vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data; vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data; th0->checksum = 0; + + if (PREDICT_FALSE + (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6))) + tcp_output_handle_link_local (tc0, b0, &next0, &error0); } /* Filter out DUPACKs if there are no OOO segments left */ if (PREDICT_FALSE (vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK)) { + /* N.B. Should not filter burst of dupacks. Two issues: + * 1) dupacks open cwnd on remote peer when congested + * 2) acks leaving should have the latest rcv_wnd since the + * burst may have eaten up all of it, so only the old ones + * could be filtered. + */ if (!tcp_session_has_ooo_data (tc0)) { error0 = TCP_ERROR_FILTERED_DUPACKS; @@ -1867,10 +1943,6 @@ tcp46_output_inline (vlib_main_t * vm, vnet_buffer (b0)->ip.adj_index[VLIB_TX] = tc0->c_rmt_dpo.dpoi_index; #endif - vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0; - vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0; - - b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; done: b0->error = node->errors[error0]; if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) @@ -1969,6 +2041,7 @@ tcp_push_header (transport_connection_t * tconn, vlib_buffer_t * b) tc->rtt_ts = tcp_time_now (); tc->rtt_seq = tc->snd_nxt; } + tcp_trajectory_add_start (b, 3); return 0; } @@ -2031,7 +2104,7 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } /* Prepare to send to IP lookup */ - vnet_buffer (b0)->sw_if_index[VLIB_TX] = 0; + vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0; next0 = TCP_RESET_NEXT_IP_LOOKUP; done: