X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp_output.c;h=74fc15fe6cca02e90438abc5e2a0301c46aa1457;hb=df0847831715ef0f639a6d8e79468cd1e2341cd6;hp=78f7c3f8294876a5df41e9ed3d8da4102c3f0044;hpb=776f3d85ebd4edfb70b0f748890f1efd98d8474c;p=vpp.git diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c index 78f7c3f8294..74fc15fe6cc 100644 --- a/src/vnet/tcp/tcp_output.c +++ b/src/vnet/tcp/tcp_output.c @@ -215,7 +215,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts) *data++ = TCP_OPTION_MSS; *data++ = TCP_OPTION_LEN_MSS; buf = clib_host_to_net_u16 (opts->mss); - clib_memcpy (data, &buf, sizeof (opts->mss)); + clib_memcpy_fast (data, &buf, sizeof (opts->mss)); data += sizeof (opts->mss); opts_len += TCP_OPTION_LEN_MSS; } @@ -240,10 +240,10 @@ tcp_options_write (u8 * data, tcp_options_t * opts) *data++ = TCP_OPTION_TIMESTAMP; *data++ = TCP_OPTION_LEN_TIMESTAMP; buf = clib_host_to_net_u32 (opts->tsval); - clib_memcpy (data, &buf, sizeof (opts->tsval)); + clib_memcpy_fast (data, &buf, sizeof (opts->tsval)); data += sizeof (opts->tsval); buf = clib_host_to_net_u32 (opts->tsecr); - clib_memcpy (data, &buf, sizeof (opts->tsecr)); + clib_memcpy_fast (data, &buf, sizeof (opts->tsecr)); data += sizeof (opts->tsecr); opts_len += TCP_OPTION_LEN_TIMESTAMP; } @@ -261,10 +261,10 @@ tcp_options_write (u8 * data, tcp_options_t * opts) for (i = 0; i < n_sack_blocks; i++) { buf = clib_host_to_net_u32 (opts->sacks[i].start); - clib_memcpy (data, &buf, seq_len); + clib_memcpy_fast (data, &buf, seq_len); data += seq_len; buf = clib_host_to_net_u32 (opts->sacks[i].end); - clib_memcpy (data, &buf, seq_len); + clib_memcpy_fast (data, &buf, seq_len); data += seq_len; } opts_len += 2 + n_sack_blocks * TCP_OPTION_LEN_SACK_BLOCK; @@ -559,7 +559,6 @@ tcp_make_ack (tcp_connection_t * tc, vlib_buffer_t * b) tcp_reuse_buffer (vm, b); tcp_make_ack_i (tc, b, TCP_STATE_ESTABLISHED, TCP_FLAG_ACK); TCP_EVT_DBG (TCP_EVT_ACK_SENT, tc); - vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK; tc->rcv_las = tc->rcv_nxt; } @@ -631,7 +630,6 @@ tcp_make_synack (tcp_connection_t * tc, vlib_buffer_t * b) tcp_options_write ((u8 *) (th + 1), snd_opts); vnet_buffer (b)->tcp.connection_index = tc->c_c_index; - vnet_buffer (b)->tcp.flags = TCP_BUF_FLAG_ACK; /* Init retransmit timer. Use update instead of set because of * retransmissions */ @@ -766,8 +764,8 @@ tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0, { ih6 = vlib_buffer_get_current (b0); ASSERT ((ih6->ip_version_traffic_class_and_flow_label & 0xF0) == 0x60); - clib_memcpy (&src_ip60, &ih6->src_address, sizeof (ip6_address_t)); - clib_memcpy (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&src_ip60, &ih6->src_address, sizeof (ip6_address_t)); + clib_memcpy_fast (&dst_ip60, &ih6->dst_address, sizeof (ip6_address_t)); } src_port = th0->src_port; @@ -1002,7 +1000,7 @@ tcp_send_syn (tcp_connection_t * tc) tcp_make_syn (tc, b); /* Measure RTT with this */ - tc->rtt_ts = tcp_time_now (); + tc->rtt_ts = tcp_time_now_us (vlib_num_workers ()? 1 : 0); tc->rtt_seq = tc->snd_nxt; tc->rto_boff = 0; @@ -1011,6 +1009,24 @@ tcp_send_syn (tcp_connection_t * tc) TCP_EVT_DBG (TCP_EVT_SYN_SENT, tc); } +void +tcp_send_synack (tcp_connection_t * tc) +{ + tcp_worker_ctx_t *wrk = tcp_get_worker (tc->c_thread_index); + vlib_main_t *vm = wrk->vm; + vlib_buffer_t *b; + u32 bi; + + /* Get buffer */ + if (PREDICT_FALSE (tcp_get_free_buffer_index (wrk, &bi))) + return; + + tc->rtt_ts = tcp_time_now_us (tc->c_thread_index); + b = vlib_get_buffer (vm, bi); + tcp_make_synack (tc, b); + tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); +} + /** * Flush tx frame populated by retransmits and timer pops */ @@ -1160,9 +1176,9 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b, if (maybe_burst) { - clib_memcpy ((u8 *) (th + 1), - tm->wrk_ctx[tc->c_thread_index].cached_opts, - tc->snd_opts_len); + clib_memcpy_fast ((u8 *) (th + 1), + tm->wrk_ctx[tc->c_thread_index].cached_opts, + tc->snd_opts_len); } else { @@ -1186,8 +1202,7 @@ tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b) tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, /* compute opts */ 0, /* burst */ 1); tc->snd_una_max = tc->snd_nxt; - ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd - + tcp_fastrecovery_sent_1_smss (tc) * tc->snd_mss)); + ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd)); tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una); /* If not tracking an ACK, start tracking */ if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc)) @@ -1223,6 +1238,56 @@ tcp_send_ack (tcp_connection_t * tc) tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); } +void +tcp_program_ack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc) +{ + if (!(tc->flags & TCP_CONN_SNDACK)) + { + vec_add1 (wrk->pending_acks, tc->c_c_index); + tc->flags |= TCP_CONN_SNDACK; + } +} + +void +tcp_program_dupack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc) +{ + if (!(tc->flags & TCP_CONN_SNDACK)) + { + vec_add1 (wrk->pending_acks, tc->c_c_index); + tc->flags |= TCP_CONN_SNDACK; + } + if (tc->pending_dupacks < 255) + tc->pending_dupacks += 1; +} + +void +tcp_send_acks (tcp_worker_ctx_t * wrk) +{ + u32 thread_index, *pending_acks; + tcp_connection_t *tc; + int i, j, n_acks; + + if (!vec_len (wrk->pending_acks)) + return; + + thread_index = wrk->vm->thread_index; + pending_acks = wrk->pending_acks; + for (i = 0; i < vec_len (pending_acks); i++) + { + tc = tcp_connection_get (pending_acks[i], thread_index); + tc->flags &= ~TCP_CONN_SNDACK; + n_acks = clib_max (1, tc->pending_dupacks); + /* If we're supposed to send dupacks but have no ooo data + * send only one ack */ + if (tc->pending_dupacks && !vec_len (tc->snd_sacks)) + n_acks = 1; + for (j = 0; j < n_acks; j++) + tcp_send_ack (tc); + tc->pending_dupacks = 0; + } + _vec_len (wrk->pending_acks) = 0; +} + /** * Delayed ack timer handler * @@ -1240,49 +1305,27 @@ tcp_timer_delack_handler (u32 index) } /** - * Build a retransmit segment + * Allocate a new buffer and build a new tcp segment * - * @return the number of bytes in the segment or 0 if there's nothing to - * retransmit + * @param wrk tcp worker + * @param tc connection for which the segment will be allocated + * @param offset offset of the first byte in the tx fifo + * @param max_deq_byte segment size + * @param[out] b pointer to buffer allocated + * + * @return the number of bytes in the segment or 0 if buffer cannot be + * allocated or no data available */ -static u32 -tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk, - tcp_connection_t * tc, u32 offset, - u32 max_deq_bytes, vlib_buffer_t ** b) +static int +tcp_prepare_segment (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, + u32 offset, u32 max_deq_bytes, vlib_buffer_t ** b) { u32 bytes_per_buffer = vnet_get_tcp_main ()->bytes_per_buffer; + u32 bi, seg_size; vlib_main_t *vm = wrk->vm; int n_bytes = 0; - u32 start, bi, available_bytes, seg_size; u8 *data; - ASSERT (tc->state >= TCP_STATE_ESTABLISHED); - ASSERT (max_deq_bytes != 0); - - /* - * Make sure we can retransmit something - */ - available_bytes = session_tx_fifo_max_dequeue (&tc->connection); - ASSERT (available_bytes >= offset); - available_bytes -= offset; - if (!available_bytes) - return 0; - max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes); - max_deq_bytes = clib_min (available_bytes, max_deq_bytes); - - /* Start is beyond snd_congestion */ - start = tc->snd_una + offset; - if (seq_geq (start, tc->snd_congestion)) - goto done; - - /* Don't overshoot snd_congestion */ - if (seq_gt (start + max_deq_bytes, tc->snd_congestion)) - { - max_deq_bytes = tc->snd_congestion - start; - if (max_deq_bytes == 0) - goto done; - } - seg_size = max_deq_bytes + MAX_HDRS_LEN; /* @@ -1330,7 +1373,7 @@ tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk, } } - tcp_get_free_buffer_index (wrk, &bi); + (void) tcp_get_free_buffer_index (wrk, &bi); ASSERT (bi != (u32) ~ 0); *b = vlib_get_buffer (vm, bi); data = tcp_init_buffer (vm, *b); @@ -1374,6 +1417,55 @@ tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk, ASSERT (n_bytes > 0); ASSERT (((*b)->current_data + (*b)->current_length) <= bytes_per_buffer); + return n_bytes; +} + +/** + * Build a retransmit segment + * + * @return the number of bytes in the segment or 0 if there's nothing to + * retransmit + */ +static u32 +tcp_prepare_retransmit_segment (tcp_worker_ctx_t * wrk, + tcp_connection_t * tc, u32 offset, + u32 max_deq_bytes, vlib_buffer_t ** b) +{ + u32 start, available_bytes; + int n_bytes = 0; + + ASSERT (tc->state >= TCP_STATE_ESTABLISHED); + ASSERT (max_deq_bytes != 0); + + /* + * Make sure we can retransmit something + */ + available_bytes = session_tx_fifo_max_dequeue (&tc->connection); + ASSERT (available_bytes >= offset); + available_bytes -= offset; + if (!available_bytes) + return 0; + + max_deq_bytes = clib_min (tc->snd_mss, max_deq_bytes); + max_deq_bytes = clib_min (available_bytes, max_deq_bytes); + + /* Start is beyond snd_congestion */ + start = tc->snd_una + offset; + if (seq_geq (start, tc->snd_congestion)) + goto done; + + /* Don't overshoot snd_congestion */ + if (seq_gt (start + max_deq_bytes, tc->snd_congestion)) + { + max_deq_bytes = tc->snd_congestion - start; + if (max_deq_bytes == 0) + goto done; + } + + n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b); + if (!n_bytes) + return 0; + if (tcp_in_fastrecovery (tc)) tc->snd_rxt_bytes += n_bytes; @@ -1424,7 +1516,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { tc = tcp_half_open_connection_get (index); /* Note: the connection may have transitioned to ESTABLISHED... */ - if (PREDICT_FALSE (tc == 0)) + if (PREDICT_FALSE (tc == 0 || tc->state != TCP_STATE_SYN_SENT)) return; tc->timers[TCP_TIMER_RETRANSMIT_SYN] = TCP_TIMER_HANDLE_INVALID; } @@ -1432,7 +1524,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn) { tc = tcp_connection_get (index, thread_index); /* Note: the connection may have been closed and pool_put */ - if (PREDICT_FALSE (tc == 0)) + if (PREDICT_FALSE (tc == 0 || tc->state < TCP_STATE_SYN_RCVD)) return; tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID; } @@ -1616,7 +1708,7 @@ tcp_timer_persist_handler (u32 index) /* Problem already solved or worse */ if (tc->state == TCP_STATE_CLOSED || tc->state > TCP_STATE_ESTABLISHED - || tc->snd_wnd > tc->snd_mss || tcp_in_recovery (tc)) + || tc->snd_wnd > tc->snd_mss) return; available_bytes = session_tx_fifo_max_dequeue (&tc->connection); @@ -1696,6 +1788,36 @@ tcp_retransmit_first_unacked (tcp_worker_ctx_t * wrk, tcp_connection_t * tc) return 0; } +static int +tcp_fast_retransmit_unsent (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, + u32 burst_size) +{ + u32 offset, n_segs = 0, n_written, bi; + vlib_main_t *vm = wrk->vm; + vlib_buffer_t *b = 0; + + tc->snd_nxt = tc->snd_una_max; + offset = tc->snd_una_max - tc->snd_una; + while (n_segs < burst_size) + { + n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b); + if (!n_written) + goto done; + + bi = vlib_get_buffer_index (vm, b); + tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); + offset += n_written; + n_segs += 1; + } + +done: + return n_segs; +} + +#define scoreboard_rescue_rxt_valid(_sb, _tc) \ + (seq_geq (_sb->rescue_rxt, _tc->snd_una) \ + && seq_leq (_sb->rescue_rxt, _tc->snd_congestion)) + /** * Do fast retransmit with SACKs */ @@ -1703,55 +1825,54 @@ int tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, u32 burst_size) { + u32 n_written = 0, offset, max_bytes, n_segs = 0, n_segs_now; + sack_scoreboard_hole_t *hole; vlib_main_t *vm = wrk->vm; - u32 n_written = 0, offset, max_bytes, n_segs = 0; vlib_buffer_t *b = 0; - sack_scoreboard_hole_t *hole; sack_scoreboard_t *sb; u32 bi, old_snd_nxt; int snd_space; + u32 max_deq; u8 snd_limited = 0, can_rescue = 0; ASSERT (tcp_in_fastrecovery (tc)); - old_snd_nxt = tc->snd_nxt; - sb = &tc->sack_sb; snd_space = tcp_available_cc_snd_space (tc); - hole = scoreboard_get_hole (sb, sb->cur_rxt_hole); - if (snd_space < tc->snd_mss) { tcp_program_fastretransmit (wrk, tc); - goto done; + return 0; } TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0); + old_snd_nxt = tc->snd_nxt; + sb = &tc->sack_sb; + hole = scoreboard_get_hole (sb, sb->cur_rxt_hole); + + max_deq = session_tx_fifo_max_dequeue (&tc->connection); + max_deq -= tc->snd_una_max - tc->snd_una; + while (snd_space > 0 && n_segs < burst_size) { - hole = scoreboard_next_rxt_hole (sb, hole, - tcp_fastrecovery_sent_1_smss (tc), - &can_rescue, &snd_limited); + hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue, + &snd_limited); if (!hole) { - if (!can_rescue || !(seq_lt (sb->rescue_rxt, tc->snd_una) - || seq_gt (sb->rescue_rxt, - tc->snd_congestion))) + if (max_deq) { - if (tcp_fastrecovery_first (tc)) - break; - - /* We tend to lose the first segment. Try re-resending - * it but only once and after we've tried everything */ - hole = scoreboard_first_hole (sb); - if (hole && hole->start == tc->snd_una) - { - tcp_retransmit_first_unacked (wrk, tc); - tcp_fastrecovery_first_on (tc); - n_segs += 1; - } - break; + snd_space = clib_min (max_deq, snd_space); + burst_size = clib_min (burst_size - n_segs, + snd_space / tc->snd_mss); + n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size); + if (max_deq > n_segs_now * tc->snd_mss) + tcp_program_fastretransmit (wrk, tc); + n_segs += n_segs_now; + goto done; } + if (!can_rescue || scoreboard_rescue_rxt_valid (sb, tc)) + break; + /* If rescue rxt undefined or less than snd_una then one segment of * up to SMSS octets that MUST include the highest outstanding * unSACKed sequence number SHOULD be returned, and RescueRxt set to @@ -1778,19 +1899,21 @@ tcp_fast_retransmit_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, max_bytes = snd_limited ? clib_min (max_bytes, tc->snd_mss) : max_bytes; if (max_bytes == 0) break; + offset = sb->high_rxt - tc->snd_una; tc->snd_nxt = sb->high_rxt; n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes, &b); + ASSERT (n_written <= snd_space); /* Nothing left to retransmit */ if (n_written == 0) break; bi = vlib_get_buffer_index (vm, b); - sb->high_rxt += n_written; tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); - ASSERT (n_written <= snd_space); + + sb->high_rxt += n_written; snd_space -= n_written; n_segs += 1; } @@ -1811,24 +1934,26 @@ int tcp_fast_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, u32 burst_size) { - u32 n_written = 0, offset = 0, bi, old_snd_nxt; + u32 n_written = 0, offset = 0, bi, old_snd_nxt, max_deq, n_segs_now; vlib_main_t *vm = wrk->vm; int snd_space, n_segs = 0; vlib_buffer_t *b; ASSERT (tcp_in_fastrecovery (tc)); TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0); - - /* Start resending from first un-acked segment */ old_snd_nxt = tc->snd_nxt; - tc->snd_nxt = tc->snd_una; - snd_space = tcp_available_cc_snd_space (tc); + if (!tcp_fastrecovery_first (tc)) + goto send_unsent; + + /* RFC 6582: [If a partial ack], retransmit the first unacknowledged + * segment. */ + snd_space = tc->sack_sb.last_bytes_delivered; + tc->snd_nxt = tc->snd_una; while (snd_space > 0 && n_segs < burst_size) { - offset += n_written; - n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, snd_space, - &b); + n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, + tc->snd_mss, &b); /* Nothing left to retransmit */ if (n_written == 0) @@ -1837,16 +1962,37 @@ tcp_fast_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, bi = vlib_get_buffer_index (vm, b); tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4); snd_space -= n_written; + offset += n_written; n_segs += 1; } - /* More data to resend */ - if (seq_lt (tc->snd_nxt, tc->snd_congestion)) - tcp_program_fastretransmit (wrk, tc); + if (n_segs == burst_size) + goto done; - /* Restore snd_nxt. If window allows, send 1 SMSS of new data */ +send_unsent: + + /* RFC 6582: Send a new segment if permitted by the new value of cwnd. */ + snd_space = tcp_available_cc_snd_space (tc); + if (snd_space < tc->snd_mss || tc->snd_mss == 0) + goto done; + + max_deq = session_tx_fifo_max_dequeue (&tc->connection); + max_deq -= tc->snd_una_max - tc->snd_una; + if (max_deq) + { + snd_space = clib_min (max_deq, snd_space); + burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss); + n_segs_now = tcp_fast_retransmit_unsent (wrk, tc, burst_size); + if (max_deq > n_segs_now * tc->snd_mss) + tcp_program_fastretransmit (wrk, tc); + n_segs += n_segs_now; + } + + /* Restore snd_nxt */ tc->snd_nxt = old_snd_nxt; +done: + tcp_fastrecovery_first_off (tc); return n_segs; } @@ -1863,13 +2009,6 @@ tcp_fast_retransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc, return tcp_fast_retransmit_no_sack (wrk, tc, burst_size); } -static u32 -tcp_session_has_ooo_data (tcp_connection_t * tc) -{ - stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index); - return svm_fifo_has_ooo_data (s->server_rx_fifo); -} - static void tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0, u16 * next0, u32 * error0) @@ -1920,8 +2059,8 @@ tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node, tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index, vm->thread_index); t = vlib_add_trace (vm, node, b, sizeof (*t)); - clib_memcpy (&t->tcp_header, th, sizeof (t->tcp_header)); - clib_memcpy (&t->tcp_connection, tc, sizeof (t->tcp_connection)); + clib_memcpy_fast (&t->tcp_header, th, sizeof (t->tcp_header)); + clib_memcpy_fast (&t->tcp_connection, tc, sizeof (t->tcp_connection)); } } @@ -1974,25 +2113,6 @@ tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0, tcp_output_handle_link_local (tc0, b0, next0, error0); } - /* Filter out DUPACKs if there are no OOO segments left */ - if (PREDICT_FALSE (vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK)) - { - /* N.B. Should not filter burst of dupacks. Two issues: - * 1) dupacks open cwnd on remote peer when congested - * 2) acks leaving should have the latest rcv_wnd since the - * burst may have eaten up all of it, so only the old ones - * could be filtered. - */ - if (!tcp_session_has_ooo_data (tc0)) - { - *error0 = TCP_ERROR_FILTERED_DUPACKS; - *next0 = TCP_OUTPUT_NEXT_DROP; - return; - } - } - - /* Stop DELACK timer and fix flags */ - tc0->flags &= ~(TCP_CONN_SNDACK); if (!TCP_ALWAYS_ACK) tcp_timer_reset (tc0, TCP_TIMER_DELACK); } @@ -2090,19 +2210,21 @@ tcp6_output (vlib_main_t * vm, vlib_node_runtime_t * node, /* *INDENT-OFF* */ VLIB_REGISTER_NODE (tcp4_output_node) = { - .function = tcp4_output,.name = "tcp4-output", - /* Takes a vector of packets. */ - .vector_size = sizeof (u32), - .n_errors = TCP_N_ERROR, - .error_strings = tcp_error_strings, - .n_next_nodes = TCP_OUTPUT_N_NEXT, - .next_nodes = { + .function = tcp4_output, + .name = "tcp4-output", + /* Takes a vector of packets. */ + .vector_size = sizeof (u32), + .n_errors = TCP_N_ERROR, + .protocol_hint = VLIB_NODE_PROTO_HINT_TCP, + .error_strings = tcp_error_strings, + .n_next_nodes = TCP_OUTPUT_N_NEXT, + .next_nodes = { #define _(s,n) [TCP_OUTPUT_NEXT_##s] = n, foreach_tcp4_output_next #undef _ - }, - .format_buffer = format_tcp_header, - .format_trace = format_tcp_tx_trace, + }, + .format_buffer = format_tcp_header, + .format_trace = format_tcp_tx_trace, }; /* *INDENT-ON* */ @@ -2116,6 +2238,7 @@ VLIB_REGISTER_NODE (tcp6_output_node) = /* Takes a vector of packets. */ .vector_size = sizeof (u32), .n_errors = TCP_N_ERROR, + .protocol_hint = VLIB_NODE_PROTO_HINT_TCP, .error_strings = tcp_error_strings, .n_next_nodes = TCP_OUTPUT_N_NEXT, .next_nodes = { @@ -2203,7 +2326,8 @@ tcp46_send_reset_inline (vlib_main_t * vm, vlib_node_runtime_t * node, else th0 = ip6_next_header ((ip6_header_t *) th0); t0 = vlib_add_trace (vm, node, b0, sizeof (*t0)); - clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header)); + clib_memcpy_fast (&t0->tcp_header, th0, + sizeof (t0->tcp_header)); } vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,