/*
* Figure out how much space we have available
*/
- available_space = stream_session_max_rx_enqueue (&tc->connection);
- max_fifo = stream_session_rx_fifo_size (&tc->connection);
+ available_space = transport_max_rx_enqueue (&tc->connection);
+ max_fifo = transport_rx_fifo_size (&tc->connection);
ASSERT (tc->rcv_opts.mss < max_fifo);
if (available_space < tc->rcv_opts.mss && available_space < max_fifo >> 3)
{
case TCP_STATE_ESTABLISHED:
case TCP_STATE_FIN_WAIT_1:
+ case TCP_STATE_CLOSED:
+ case TCP_STATE_CLOSE_WAIT:
return tcp_make_established_options (tc, opts);
case TCP_STATE_SYN_RCVD:
return tcp_make_synack_options (tc, opts);
tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs,
u32 wanted)
{
+ tcp_worker_ctx_t *ctx = &tm->wrk_ctx[thread_index];
vlib_main_t *vm = vlib_get_main ();
u32 n_alloc;
ASSERT (wanted > *n_bufs);
- vec_validate_aligned (tm->tx_buffers[thread_index], wanted - 1,
- CLIB_CACHE_LINE_BYTES);
- n_alloc = vlib_buffer_alloc (vm, &tm->tx_buffers[thread_index][*n_bufs],
+ vec_validate_aligned (ctx->tx_buffers, wanted - 1, CLIB_CACHE_LINE_BYTES);
+ n_alloc = vlib_buffer_alloc (vm, &ctx->tx_buffers[*n_bufs],
wanted - *n_bufs);
*n_bufs += n_alloc;
- _vec_len (tm->tx_buffers[thread_index]) = *n_bufs;
+ _vec_len (ctx->tx_buffers) = *n_bufs;
return n_alloc;
}
tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx)
{
u32 thread_index = vlib_get_thread_index ();
- u16 n_bufs = vec_len (tm->tx_buffers[thread_index]);
+ tcp_worker_ctx_t *ctx = &tm->wrk_ctx[thread_index];
+ u16 n_bufs = vec_len (ctx->tx_buffers);
TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (thread_index);
return -1;
}
}
- *bidx = tm->tx_buffers[thread_index][--n_bufs];
- _vec_len (tm->tx_buffers[thread_index]) = n_bufs;
+ *bidx = ctx->tx_buffers[--n_bufs];
+ _vec_len (ctx->tx_buffers) = n_bufs;
return 0;
}
next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
tcp_trajectory_add_start (b, 1);
- f = tm->ip_lookup_tx_frames[!is_ip4][thread_index];
+ f = tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4];
if (!f)
{
f = vlib_get_frame_to_node (vm, next_index);
ASSERT (f);
- tm->ip_lookup_tx_frames[!is_ip4][thread_index] = f;
+ tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = f;
}
to_next = vlib_frame_vector_args (f);
if (flush || f->n_vectors == VLIB_FRAME_SIZE)
{
vlib_put_frame_to_node (vm, next_index, f);
- tm->ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
+ tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = 0;
}
}
tcp_trajectory_add_start (b, 2);
/* Get frame to v4/6 output node */
- f = tm->tx_frames[!is_ip4][thread_index];
+ f = tm->wrk_ctx[thread_index].tx_frames[!is_ip4];
if (!f)
{
f = vlib_get_frame_to_node (vm, next_index);
ASSERT (f);
- tm->tx_frames[!is_ip4][thread_index] = f;
+ tm->wrk_ctx[thread_index].tx_frames[!is_ip4] = f;
}
to_next = vlib_frame_vector_args (f);
to_next[f->n_vectors] = bi;
if (flush || f->n_vectors == VLIB_FRAME_SIZE)
{
vlib_put_frame_to_node (vm, next_index, f);
- tm->tx_frames[!is_ip4][thread_index] = 0;
+ tm->wrk_ctx[thread_index].tx_frames[!is_ip4] = 0;
}
}
void
tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
{
- if (tcp_main.tx_frames[!is_ip4][thread_index])
+ if (tcp_main.wrk_ctx[thread_index].tx_frames[!is_ip4])
{
u32 next_index;
next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
vlib_put_frame_to_node (vm, next_index,
- tcp_main.tx_frames[!is_ip4][thread_index]);
- tcp_main.tx_frames[!is_ip4][thread_index] = 0;
+ tcp_main.
+ wrk_ctx[thread_index].tx_frames[!is_ip4]);
+ tcp_main.wrk_ctx[thread_index].tx_frames[!is_ip4] = 0;
}
}
always_inline void
tcp_flush_frame_to_ip_lookup (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
{
- if (tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index])
+ if (tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4])
{
u32 next_index;
next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
vlib_put_frame_to_node (vm, next_index,
- tcp_main.ip_lookup_tx_frames[!is_ip4]
- [thread_index]);
- tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
+ tcp_main.
+ wrk_ctx[thread_index].ip_lookup_tx_frames
+ [!is_ip4]);
+ tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = 0;
}
}
switch (next_state)
{
case TCP_STATE_ESTABLISHED:
+ case TCP_STATE_CLOSE_WAIT:
return TCP_FLAG_ACK;
case TCP_STATE_SYN_RCVD:
return TCP_FLAG_SYN | TCP_FLAG_ACK;
tc->snd_nxt += data_len;
tc->rcv_las = tc->rcv_nxt;
- /* TODO this is updated in output as well ... */
- if (seq_gt (tc->snd_nxt, tc->snd_una_max))
- {
- tc->snd_una_max = tc->snd_nxt;
- tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
- }
-
TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
}
/*
* Make sure we can retransmit something
*/
- available_bytes = stream_session_tx_fifo_max_dequeue (&tc->connection);
+ available_bytes = session_tx_fifo_max_dequeue (&tc->connection);
ASSERT (available_bytes >= offset);
available_bytes -= offset;
if (!available_bytes)
/* Make sure we have enough buffers */
n_bufs_per_seg = ceil ((double) seg_size / tm->bytes_per_buffer);
- available_bufs = vec_len (tm->tx_buffers[thread_index]);
+ available_bufs = vec_len (tm->wrk_ctx[thread_index].tx_buffers);
if (n_bufs_per_seg > available_bufs)
{
tcp_alloc_tx_buffers (tm, thread_index, &available_bufs,
* Reset congestion control, switch cwnd to loss window and try again.
*/
static void
-tcp_rtx_timeout_cc (tcp_connection_t * tc)
+tcp_rxt_timeout_cc (tcp_connection_t * tc)
{
+ TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 6);
tc->prev_ssthresh = tc->ssthresh;
tc->prev_cwnd = tc->cwnd;
tcp_cc_fastrecovery_exit (tc);
/* Start again from the beginning */
- tc->ssthresh = clib_max (tcp_flight_size (tc) / 2, 2 * tc->snd_mss);
+ tc->cc_algo->congestion (tc);
tc->cwnd = tcp_loss_wnd (tc);
tc->snd_congestion = tc->snd_una_max;
tc->rtt_ts = 0;
+ tc->cwnd_acc_bytes = 0;
+
tcp_recovery_on (tc);
}
tc->timers[TCP_TIMER_RETRANSMIT] = TCP_TIMER_HANDLE_INVALID;
}
+ TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
+
if (tc->state >= TCP_STATE_ESTABLISHED)
{
/* Lost FIN, retransmit and return */
}
/* Shouldn't be here */
- if (tc->snd_una == tc->snd_una_max)
+ if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_una_max)
+ || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)))
{
tcp_recovery_off (tc);
return;
/* First retransmit timeout */
if (tc->rto_boff == 1)
- tcp_rtx_timeout_cc (tc);
+ tcp_rxt_timeout_cc (tc);
- tc->snd_nxt = tc->snd_una;
+ tc->snd_una_max = tc->snd_nxt = tc->snd_una;
tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
- TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 1);
-
/* Send one segment. Note that n_bytes may be zero due to buffer shortfall */
n_bytes = tcp_prepare_retransmit_segment (tc, 0, tc->snd_mss, &b);
|| tc->snd_wnd > tc->snd_mss || tcp_in_recovery (tc))
return;
- available_bytes = stream_session_tx_fifo_max_dequeue (&tc->connection);
+ available_bytes = session_tx_fifo_max_dequeue (&tc->connection);
offset = tc->snd_una_max - tc->snd_una;
/* Reprogram persist if no new bytes available to send. We may have data
|| tc->rto_boff > 1));
tcp_push_hdr_i (tc, b, tc->state, 0);
+ tc->snd_una_max = tc->snd_nxt;
+ tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
/* Just sent new data, enable retransmit */
tcp_fast_retransmit_sack (tcp_connection_t * tc)
{
vlib_main_t *vm = vlib_get_main ();
- u32 n_written = 0, offset, max_bytes;
+ u32 n_written = 0, offset, max_bytes, n_segs = 0;
vlib_buffer_t *b = 0;
sack_scoreboard_hole_t *hole;
sack_scoreboard_t *sb;
u8 snd_limited = 0, can_rescue = 0;
ASSERT (tcp_in_fastrecovery (tc));
- TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
old_snd_nxt = tc->snd_nxt;
sb = &tc->sack_sb;
- snd_space = tcp_available_snd_space (tc);
+ snd_space = tcp_available_cc_snd_space (tc);
+ if (snd_space < tc->snd_mss)
+ goto done;
+
+ TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 0);
hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
- while (hole && snd_space > 0)
+ while (hole && snd_space > 0 && n_segs++ < VLIB_FRAME_SIZE)
{
hole = scoreboard_next_rxt_hole (sb, hole,
tcp_fastrecovery_sent_1_smss (tc),
/* Start resending from first un-acked segment */
old_snd_nxt = tc->snd_nxt;
tc->snd_nxt = tc->snd_una;
- snd_space = tcp_available_snd_space (tc);
+ snd_space = tcp_available_cc_snd_space (tc);
while (snd_space > 0)
{
void
tcp_fast_retransmit (tcp_connection_t * tc)
{
- if (tcp_opts_sack_permitted (&tc->rcv_opts)
- && scoreboard_first_hole (&tc->sack_sb))
+ if (tcp_opts_sack_permitted (&tc->rcv_opts))
tcp_fast_retransmit_sack (tc);
else
tcp_fast_retransmit_no_sack (tc);
return svm_fifo_has_ooo_data (s->server_rx_fifo);
}
-typedef struct
-{
- fib_protocol_t nh_proto;
- vnet_link_t link_type;
- ip46_address_t ip;
- u32 sw_if_index;
-} tcp_adj_add_args_t;
-
-void
-tcp_output_add_adj (tcp_adj_add_args_t * args)
-{
- adj_nbr_add_or_lock (args->nh_proto, args->link_type, &args->ip,
- args->sw_if_index);
-}
-
static void
tcp_output_handle_link_local (tcp_connection_t * tc0, vlib_buffer_t * b0,
- u32 * next0, u32 * error0)
+ u16 * next0, u32 * error0)
{
ip_adjacency_t *adj;
adj_index_t ai;
+ /* Not thread safe but as long as the connection exists the adj should
+ * not be removed */
ai = adj_nbr_find (FIB_PROTOCOL_IP6, VNET_LINK_IP6, &tc0->c_rmt_ip,
tc0->sw_if_index);
if (ai == ADJ_INDEX_INVALID)
{
- tcp_adj_add_args_t args = {
- .nh_proto = FIB_PROTOCOL_IP6,
- .link_type = VNET_LINK_IP6,
- .ip = tc0->c_rmt_ip,
- .sw_if_index = tc0->sw_if_index
- };
- vlib_rpc_call_main_thread (tcp_output_add_adj, (u8 *) & args,
- sizeof (args));
vnet_buffer (b0)->sw_if_index[VLIB_TX] = ~0;
*next0 = TCP_OUTPUT_NEXT_DROP;
*error0 = TCP_ERROR_LINK_LOCAL_RW;
}
adj = adj_get (ai);
- if (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE)
+ if (PREDICT_TRUE (adj->lookup_next_index == IP_LOOKUP_NEXT_REWRITE))
*next0 = TCP_OUTPUT_NEXT_IP_REWRITE;
else if (adj->lookup_next_index == IP_LOOKUP_NEXT_ARP)
*next0 = TCP_OUTPUT_NEXT_IP_ARP;
+ else
+ {
+ *next0 = TCP_OUTPUT_NEXT_DROP;
+ *error0 = TCP_ERROR_LINK_LOCAL_RW;
+ }
+ vnet_buffer (b0)->ip.adj_index[VLIB_TX] = ai;
}
-always_inline uword
-tcp46_output_inline (vlib_main_t * vm,
- vlib_node_runtime_t * node,
- vlib_frame_t * from_frame, int is_ip4)
+static void
+tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
+ u32 * to_next, u32 n_bufs)
{
- u32 n_left_from, next_index, *from, *to_next;
- u32 my_thread_index = vm->thread_index;
+ u32 n_trace = vlib_get_trace_count (vm, node);
+ tcp_connection_t *tc;
+ tcp_tx_trace_t *t;
+ vlib_buffer_t *b;
+ tcp_header_t *th;
+ int i;
- from = vlib_frame_vector_args (from_frame);
- n_left_from = from_frame->n_vectors;
- next_index = node->cached_next_index;
- tcp_set_time_now (my_thread_index);
+ for (i = 0; i < clib_min (n_trace, n_bufs); i++)
+ {
+ b = vlib_get_buffer (vm, to_next[i]);
+ th = vlib_buffer_get_current (b);
+ tc = tcp_connection_get (vnet_buffer (b)->tcp.connection_index,
+ vm->thread_index);
+ t = vlib_add_trace (vm, node, b, sizeof (*t));
+ clib_memcpy (&t->tcp_header, th, sizeof (t->tcp_header));
+ clib_memcpy (&t->tcp_connection, tc, sizeof (t->tcp_connection));
+ }
+}
- while (n_left_from > 0)
+static inline void
+tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
+ tcp_connection_t * tc0, u8 is_ip4)
+{
+ tcp_header_t *th0 = 0;
+
+ th0 = vlib_buffer_get_current (b0);
+ TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
+ if (is_ip4)
{
- u32 n_left_to_next;
+ vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
+ IP_PROTOCOL_TCP, 1);
+ b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
+ th0->checksum = 0;
+ }
+ else
+ {
+ ip6_header_t *ih0;
+ ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
+ &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
+ b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
+ vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
+ vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
+ th0->checksum = 0;
+ }
+}
- vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
+static inline void
+tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
+ u32 * error0, u16 * next0, u8 is_ip4)
+{
- while (n_left_from > 0 && n_left_to_next > 0)
+ if (PREDICT_FALSE (tc0->state == TCP_STATE_CLOSED))
+ {
+ *error0 = TCP_ERROR_INVALID_CONNECTION;
+ *next0 = TCP_OUTPUT_NEXT_DROP;
+ return;
+ }
+
+ vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
+ vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
+
+ if (!is_ip4)
+ {
+ if (PREDICT_FALSE (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
+ tcp_output_handle_link_local (tc0, b0, next0, error0);
+ }
+
+ /* Filter out DUPACKs if there are no OOO segments left */
+ if (PREDICT_FALSE (vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK))
+ {
+ /* N.B. Should not filter burst of dupacks. Two issues:
+ * 1) dupacks open cwnd on remote peer when congested
+ * 2) acks leaving should have the latest rcv_wnd since the
+ * burst may have eaten up all of it, so only the old ones
+ * could be filtered.
+ */
+ if (!tcp_session_has_ooo_data (tc0))
{
- u32 bi0;
- vlib_buffer_t *b0;
- tcp_connection_t *tc0;
- tcp_tx_trace_t *t0;
- tcp_header_t *th0 = 0;
- u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_IP_LOOKUP;
+ *error0 = TCP_ERROR_FILTERED_DUPACKS;
+ *next0 = TCP_OUTPUT_NEXT_DROP;
+ return;
+ }
+ }
- if (n_left_from > 1)
- {
- vlib_buffer_t *pb;
- pb = vlib_get_buffer (vm, from[1]);
- vlib_prefetch_buffer_header (pb, STORE);
- CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
- }
+ /* Stop DELACK timer and fix flags */
+ tc0->flags &= ~(TCP_CONN_SNDACK);
+ if (!TCP_ALWAYS_ACK)
+ tcp_timer_reset (tc0, TCP_TIMER_DELACK);
+}
- bi0 = from[0];
- to_next[0] = bi0;
- from += 1;
- to_next += 1;
- n_left_from -= 1;
- n_left_to_next -= 1;
+always_inline uword
+tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+ vlib_frame_t * frame, int is_ip4)
+{
+ u32 n_left_from, *from, thread_index = vm->thread_index;
+ vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
+ u16 nexts[VLIB_FRAME_SIZE], *next;
- b0 = vlib_get_buffer (vm, bi0);
- tc0 = tcp_connection_get (vnet_buffer (b0)->tcp.connection_index,
- my_thread_index);
- if (PREDICT_FALSE (tc0 == 0 || tc0->state == TCP_STATE_CLOSED))
- {
- error0 = TCP_ERROR_INVALID_CONNECTION;
- next0 = TCP_OUTPUT_NEXT_DROP;
- goto done;
- }
+ from = vlib_frame_vector_args (frame);
+ n_left_from = frame->n_vectors;
+ tcp_set_time_now (thread_index);
- th0 = vlib_buffer_get_current (b0);
- TCP_EVT_DBG (TCP_EVT_OUTPUT, tc0, th0->flags, b0->current_length);
- vnet_buffer (b0)->sw_if_index[VLIB_TX] = tc0->c_fib_index;
- vnet_buffer (b0)->sw_if_index[VLIB_RX] = 0;
+ if (PREDICT_FALSE (node->flags & VLIB_NODE_FLAG_TRACE))
+ tcp46_output_trace_frame (vm, node, from, n_left_from);
- if (is_ip4)
- {
- vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
- IP_PROTOCOL_TCP, 1);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
- th0->checksum = 0;
- }
- else
- {
- ip6_header_t *ih0;
- ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6,
- &tc0->c_rmt_ip6, IP_PROTOCOL_TCP);
- b0->flags |= VNET_BUFFER_F_OFFLOAD_TCP_CKSUM;
- vnet_buffer (b0)->l3_hdr_offset = (u8 *) ih0 - b0->data;
- vnet_buffer (b0)->l4_hdr_offset = (u8 *) th0 - b0->data;
- th0->checksum = 0;
-
- if (PREDICT_FALSE
- (ip6_address_is_link_local_unicast (&tc0->c_rmt_ip6)))
- tcp_output_handle_link_local (tc0, b0, &next0, &error0);
- }
+ vlib_get_buffers (vm, from, bufs, n_left_from);
+ b = bufs;
+ next = nexts;
- /* Filter out DUPACKs if there are no OOO segments left */
- if (PREDICT_FALSE
- (vnet_buffer (b0)->tcp.flags & TCP_BUF_FLAG_DUPACK))
- {
- /* N.B. Should not filter burst of dupacks. Two issues:
- * 1) dupacks open cwnd on remote peer when congested
- * 2) acks leaving should have the latest rcv_wnd since the
- * burst may have eaten up all of it, so only the old ones
- * could be filtered.
- */
- if (!tcp_session_has_ooo_data (tc0))
- {
- error0 = TCP_ERROR_FILTERED_DUPACKS;
- next0 = TCP_OUTPUT_NEXT_DROP;
- goto done;
- }
- }
+ while (n_left_from >= 4)
+ {
+ u32 error0 = TCP_ERROR_PKTS_SENT, error1 = TCP_ERROR_PKTS_SENT;
+ tcp_connection_t *tc0, *tc1;
- /* Stop DELACK timer and fix flags */
- tc0->flags &= ~(TCP_CONN_SNDACK);
- tcp_timer_reset (tc0, TCP_TIMER_DELACK);
+ {
+ vlib_prefetch_buffer_header (b[2], STORE);
+ CLIB_PREFETCH (b[2]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
- /* If not retransmitting
- * 1) update snd_una_max (SYN, SYNACK, FIN)
- * 2) If we're not tracking an ACK, start tracking */
- if (seq_lt (tc0->snd_una_max, tc0->snd_nxt))
- {
- tc0->snd_una_max = tc0->snd_nxt;
- if (tc0->rtt_ts == 0)
- {
- tc0->rtt_ts = tcp_time_now ();
- tc0->rtt_seq = tc0->snd_nxt;
- }
- }
+ vlib_prefetch_buffer_header (b[3], STORE);
+ CLIB_PREFETCH (b[3]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+ }
- /* Set the retransmit timer if not set already and not
- * doing a pure ACK */
- if (!tcp_timer_is_active (tc0, TCP_TIMER_RETRANSMIT)
- && tc0->snd_nxt != tc0->snd_una)
- {
- tcp_retransmit_timer_set (tc0);
- tc0->rto_boff = 0;
- }
+ next[0] = next[1] = TCP_OUTPUT_NEXT_IP_LOOKUP;
-#if 0
- /* Make sure we haven't lost route to our peer */
- if (PREDICT_FALSE (tc0->last_fib_check
- < tc0->snd_opts.tsval + TCP_FIB_RECHECK_PERIOD))
- {
- if (PREDICT_TRUE
- (tc0->c_rmt_fei == tcp_lookup_rmt_in_fib (tc0)))
- {
- tc0->last_fib_check = tc0->snd_opts.tsval;
- }
- else
- {
- clib_warning ("lost connection to peer");
- tcp_connection_reset (tc0);
- goto done;
- }
- }
+ tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
+ thread_index);
+ tc1 = tcp_connection_get (vnet_buffer (b[1])->tcp.connection_index,
+ thread_index);
- /* Use pre-computed dpo to set next node */
- next0 = tc0->c_rmt_dpo.dpoi_next_node;
- vnet_buffer (b0)->ip.adj_index[VLIB_TX] = tc0->c_rmt_dpo.dpoi_index;
-#endif
+ tcp_output_push_ip (vm, b[0], tc0, is_ip4);
+ tcp_output_push_ip (vm, b[1], tc1, is_ip4);
- done:
- b0->error = node->errors[error0];
- if (PREDICT_FALSE (b0->flags & VLIB_BUFFER_IS_TRACED))
- {
- t0 = vlib_add_trace (vm, node, b0, sizeof (*t0));
- if (th0)
- {
- clib_memcpy (&t0->tcp_header, th0, sizeof (t0->tcp_header));
- }
- else
- {
- memset (&t0->tcp_header, 0, sizeof (t0->tcp_header));
- }
- clib_memcpy (&t0->tcp_connection, tc0,
- sizeof (t0->tcp_connection));
- }
+ tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
+ tcp_output_handle_packet (tc1, b[1], &error1, &next[1], is_ip4);
- vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
- n_left_to_next, bi0, next0);
+ b += 2;
+ next += 2;
+ n_left_from -= 2;
+ }
+ while (n_left_from > 0)
+ {
+ u32 error0 = TCP_ERROR_PKTS_SENT;
+ tcp_connection_t *tc0;
+
+ if (n_left_from > 1)
+ {
+ vlib_prefetch_buffer_header (b[0], STORE);
+ CLIB_PREFETCH (b[0]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
}
- vlib_put_next_frame (vm, node, next_index, n_left_to_next);
+ next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
+ tc0 = tcp_connection_get (vnet_buffer (b[0])->tcp.connection_index,
+ thread_index);
+
+ tcp_output_push_ip (vm, b[0], tc0, is_ip4);
+ tcp_output_handle_packet (tc0, b[0], &error0, &next[0], is_ip4);
+
+ b += 1;
+ next += 1;
+ n_left_from -= 1;
}
- return from_frame->n_vectors;
+ vlib_buffer_enqueue_to_next (vm, node, from, nexts, frame->n_vectors);
+ return frame->n_vectors;
}
static uword
VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output);
u32
-tcp_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
+tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
{
- tcp_connection_t *tc;
-
- tc = (tcp_connection_t *) tconn;
+ tcp_connection_t *tc = (tcp_connection_t *) tconn;
tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0);
+ tc->snd_una_max = tc->snd_nxt;
ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
-
+ tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
+ /* If not tracking an ACK, start tracking */
if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
{
tc->rtt_ts = tcp_time_now ();
tc->rtt_seq = tc->snd_nxt;
}
+ if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
+ {
+ tcp_retransmit_timer_set (tc);
+ tc->rto_boff = 0;
+ }
tcp_trajectory_add_start (b, 3);
return 0;
}
Code Review / vpp.git / blobdiff