X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp_output.c;h=5172fcad4f430251faab64982fe95159e2bf85e0;hb=5035bf04130a35b76f6b49f450e27d02bafb9dab;hp=cfdd1da9188100b3d5c6c8624a38f02625a6a111;hpb=505fd37b31aa5a7d242184ebe281dbd4622ad277;p=vpp.git diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index cfdd1da9188..5172fcad4f4 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -321,7 +321,6 @@ tcp_update_burst_snd_vars (tcp_connection_t * tc) if (tc->snd_una == tc->snd_nxt) { tcp_cc_event (tc, TCP_CC_EVT_START_TX); - tcp_connection_tx_pacer_reset (tc, tc->cwnd, TRANSPORT_PACER_MIN_BURST); } if (tc->flags & TCP_CONN_PSH_PENDING) @@ -613,11 +612,11 @@ tcp_buffer_make_reset (vlib_main_t *vm, vlib_buffer_t *b, u8 is_ip4) /* Zero all flags but free list index and trace flag */ b->flags &= VLIB_BUFFER_NEXT_PRESENT - 1; - b->current_data = 0; + /* Make sure new tcp header comes after current ip */ + b->current_data = ((u8 *) th - b->data) + sizeof (tcp_header_t); b->current_length = 0; b->total_length_not_including_first_buffer = 0; vnet_buffer (b)->tcp.flags = 0; - vlib_buffer_make_headroom (b, TRANSPORT_MAX_HDRS_LEN); /* * Add TCP and IP headers @@ -688,6 +687,7 @@ tcp_send_reset_w_pkt (tcp_connection_t * tc, vlib_buffer_t * pkt, flags = TCP_FLAG_RST; seq = pkt_th->ack_number; ack = (tc->state >= TCP_STATE_SYN_RCVD) ? tc->rcv_nxt : 0; + ack = clib_host_to_net_u32 (ack); } else { @@ -787,7 +787,7 @@ tcp_send_syn (tcp_connection_t * tc) * such that we can return if we've ran out. */ tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, - tc->rto * TCP_TO_TIMER_TICK); + (u32) tc->rto * TCP_TO_TIMER_TICK); if (PREDICT_FALSE (!vlib_buffer_alloc (vm, &bi, 1))) { @@ -963,11 +963,9 @@ tcp_buffer_len (vlib_buffer_t * b) return data_len; } -u32 -tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) +always_inline u32 +tcp_push_one_header (tcp_connection_t *tc, vlib_buffer_t *b) { - tcp_connection_t *tc = (tcp_connection_t *) tconn; - if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE) tcp_bt_track_tx (tc, tcp_buffer_len (b)); @@ -975,6 +973,37 @@ tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b) /* update_snd_nxt */ 1); tcp_validate_txf_size (tc, tc->snd_nxt - tc->snd_una); + return 0; +} + +u32 +tcp_session_push_header (transport_connection_t *tconn, vlib_buffer_t **bs, + u32 n_bufs) +{ + tcp_connection_t *tc = (tcp_connection_t *) tconn; + + while (n_bufs >= 4) + { + vlib_prefetch_buffer_header (bs[2], STORE); + vlib_prefetch_buffer_header (bs[3], STORE); + + tcp_push_one_header (tc, bs[0]); + tcp_push_one_header (tc, bs[1]); + + n_bufs -= 2; + bs += 2; + } + while (n_bufs) + { + if (n_bufs > 1) + vlib_prefetch_buffer_header (bs[1], STORE); + + tcp_push_one_header (tc, bs[0]); + + n_bufs -= 1; + bs += 1; + } + /* If not tracking an ACK, start tracking */ if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc)) { @@ -1336,7 +1365,10 @@ tcp_timer_retransmit_handler (tcp_connection_t * tc) } if (tcp_opts_sack_permitted (&tc->rcv_opts)) - tcp_check_sack_reneging (tc); + { + tcp_check_sack_reneging (tc); + scoreboard_rxt_mark_lost (&tc->sack_sb, tc->snd_una, tc->snd_nxt); + } /* Update send congestion to make sure that rxt has data to send */ tc->snd_congestion = tc->snd_nxt; @@ -1477,7 +1509,7 @@ tcp_timer_retransmit_syn_handler (tcp_connection_t * tc) tcp_enqueue_half_open (wrk, tc, b, bi); tcp_timer_update (&wrk->timer_wheel, tc, TCP_TIMER_RETRANSMIT_SYN, - tc->rto * TCP_TO_TIMER_TICK); + (u32) tc->rto * TCP_TO_TIMER_TICK); } /** @@ -1533,8 +1565,10 @@ tcp_timer_persist_handler (tcp_connection_t * tc) tcp_validate_txf_size (tc, offset); tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state); - max_snd_bytes = clib_min (tc->snd_mss, + max_snd_bytes = clib_min (clib_min (tc->snd_mss, available_bytes), tm->bytes_per_buffer - TRANSPORT_MAX_HDRS_LEN); + if (tc->snd_wnd > 0) + max_snd_bytes = clib_min (tc->snd_wnd, max_snd_bytes); n_bytes = session_tx_fifo_peek_bytes (&tc->connection, data, offset, max_snd_bytes); b->current_length = n_bytes; @@ -1715,7 +1749,7 @@ tcp_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, && tc->rxt_head != tc->snd_una && tcp_retransmit_should_retry_head (tc, sb)) { - max_bytes = clib_min (tc->snd_mss, tc->snd_congestion - tc->snd_una); + max_bytes = clib_min (tc->snd_mss, tc->snd_nxt - tc->snd_una); n_written = tcp_prepare_retransmit_segment (wrk, tc, 0, max_bytes, &b); if (!n_written) { @@ -2157,6 +2191,7 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, u32 n_left_from, *from, thread_index = vm->thread_index; vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; u16 nexts[VLIB_FRAME_SIZE], *next; + u16 err_counters[TCP_N_ERROR] = { 0 }; from = vlib_frame_vector_args (frame); n_left_from = frame->n_vectors; @@ -2207,7 +2242,8 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } else { - b[0]->error = node->errors[TCP_ERROR_INVALID_CONNECTION]; + tcp_inc_err_counter (err_counters, TCP_ERROR_INVALID_CONNECTION, + 1); next[0] = TCP_OUTPUT_NEXT_DROP; } if (tc1 != 0) @@ -2218,7 +2254,8 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } else { - b[1]->error = node->errors[TCP_ERROR_INVALID_CONNECTION]; + tcp_inc_err_counter (err_counters, TCP_ERROR_INVALID_CONNECTION, + 1); next[1] = TCP_OUTPUT_NEXT_DROP; } } @@ -2248,7 +2285,7 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, } else { - b[0]->error = node->errors[TCP_ERROR_INVALID_CONNECTION]; + tcp_inc_err_counter (err_counters, TCP_ERROR_INVALID_CONNECTION, 1); next[0] = TCP_OUTPUT_NEXT_DROP; } @@ -2257,6 +2294,7 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node, n_left_from -= 1; } + tcp_store_err_counters (output, err_counters); vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); vlib_node_increment_counter (vm, tcp_node_index (output, is_ip4), TCP_ERROR_PKTS_SENT, frame->n_vectors); @@ -2330,76 +2368,98 @@ typedef enum _tcp_reset_next _(DROP, "error-drop") \ _(IP_LOOKUP, "ip6-lookup") +static void +tcp_reset_trace_frame (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_buffer_t **bs, u32 n_bufs, u8 is_ip4) +{ + tcp_header_t *tcp; + tcp_tx_trace_t *t; + int i; + + for (i = 0; i < n_bufs; i++) + { + if (bs[i]->flags & VLIB_BUFFER_IS_TRACED) + { + tcp = vlib_buffer_get_current (bs[i]); + t = vlib_add_trace (vm, node, bs[i], sizeof (*t)); + + if (is_ip4) + { + ip4_header_t *ih4 = vlib_buffer_get_current (bs[i]); + tcp = ip4_next_header (ih4); + t->tcp_connection.c_lcl_ip.ip4 = ih4->dst_address; + t->tcp_connection.c_rmt_ip.ip4 = ih4->src_address; + t->tcp_connection.c_is_ip4 = 1; + } + else + { + ip6_header_t *ih6 = vlib_buffer_get_current (bs[i]); + tcp = ip6_next_header (ih6); + t->tcp_connection.c_lcl_ip.ip6 = ih6->dst_address; + t->tcp_connection.c_rmt_ip.ip6 = ih6->src_address; + } + t->tcp_connection.c_lcl_port = tcp->dst_port; + t->tcp_connection.c_rmt_port = tcp->src_port; + t->tcp_connection.c_proto = TRANSPORT_PROTO_TCP; + clib_memcpy_fast (&t->tcp_header, tcp, sizeof (t->tcp_header)); + } + } +} + static uword -tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, - vlib_frame_t * from_frame, u8 is_ip4) +tcp46_reset_inline (vlib_main_t *vm, vlib_node_runtime_t *node, + vlib_frame_t *frame, u8 is_ip4) { - u32 error0 = TCP_ERROR_RST_SENT, next0 = TCP_RESET_NEXT_IP_LOOKUP; - u32 n_left_from, next_index, *from, *to_next; + vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b; + u16 nexts[VLIB_FRAME_SIZE], *next; + u32 n_left_from, *from; - from = vlib_frame_vector_args (from_frame); - n_left_from = from_frame->n_vectors; + from = vlib_frame_vector_args (frame); + n_left_from = frame->n_vectors; + vlib_get_buffers (vm, from, bufs, n_left_from); - next_index = node->cached_next_index; + b = bufs; + next = nexts; while (n_left_from > 0) { - u32 n_left_to_next; + tcp_buffer_make_reset (vm, b[0], is_ip4); - vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); + /* IP lookup in fib where it was received. Previous value + * was overwritten by tcp-input */ + vnet_buffer (b[0])->sw_if_index[VLIB_TX] = + vec_elt (ip4_main.fib_index_by_sw_if_index, + vnet_buffer (b[0])->sw_if_index[VLIB_RX]); - while (n_left_from > 0 && n_left_to_next > 0) - { - vlib_buffer_t *b0; - tcp_tx_trace_t *t0; - tcp_header_t *th0; - u32 bi0; - - bi0 = from[0]; - to_next[0] = bi0; - from += 1; - to_next += 1; - n_left_from -= 1; - n_left_to_next -= 1; - - b0 = vlib_get_buffer (vm, bi0); - tcp_buffer_make_reset (vm, b0, is_ip4); - - /* Prepare to send to IP lookup */ - vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0; - - b0->error = node->errors[error0]; - b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; - if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED)) - { - th0 = vlib_buffer_get_current (b0); - if (is_ip4) - th0 = ip4_next_header ((ip4_header_t *) th0); - else - th0 = ip6_next_header ((ip6_header_t *) th0); - t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy_fast (&t0->tcp_header, th0, - sizeof (t0->tcp_header)); - } + b[0]->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED; + next[0] = TCP_RESET_NEXT_IP_LOOKUP; - vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next, - n_left_to_next, bi0, next0); - } - vlib_put_next_frame (vm, node, next_index, n_left_to_next); + b += 1; + next += 1; + n_left_from -= 1; } - return from_frame->n_vectors; + + if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE)) + tcp_reset_trace_frame (vm, node, bufs, frame->n_vectors, is_ip4); + + vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors); + + vlib_node_increment_counter (vm, node->node_index, TCP_ERROR_RST_SENT, + frame->n_vectors); + + return frame->n_vectors; } VLIB_NODE_FN (tcp4_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame) { - return tcp46_send_reset_inline (vm, node, from_frame, 1); + return tcp46_reset_inline (vm, node, from_frame, 1); } VLIB_NODE_FN (tcp6_reset_node) (vlib_main_t * vm, vlib_node_runtime_t * node, vlib_frame_t * from_frame) { - return tcp46_send_reset_inline (vm, node, from_frame, 0); + return tcp46_reset_inline (vm, node, from_frame, 0); } /* *INDENT-OFF* */