tcp_update_rcv_wnd (tc);
if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
- tc->flags |= TCP_CONN_TRACK_BURST;
+ tcp_bt_check_app_limited (tc);
if (tc->snd_una == tc->snd_nxt)
{
return checksum;
}
-
/**
* Prepare ACK
*/
session_flush_frames_main_thread (wrk->vm);
}
-always_inline void
-tcp_enqueue_to_output_i (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
- u8 is_ip4, u8 flush)
+static void
+tcp_enqueue_to_output (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
+ u8 is_ip4)
{
- u32 *to_next, next_index;
- vlib_frame_t *f;
+ session_type_t st;
b->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
b->error = 0;
- /* Decide where to send the packet */
- next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
- tcp_trajectory_add_start (b, 2);
-
- /* Get frame to v4/6 output node */
- f = wrk->tx_frames[!is_ip4];
- if (!f)
- {
- f = vlib_get_frame_to_node (wrk->vm, next_index);
- ASSERT (f);
- wrk->tx_frames[!is_ip4] = f;
- }
- to_next = vlib_frame_vector_args (f);
- to_next[f->n_vectors] = bi;
- f->n_vectors += 1;
- if (flush || f->n_vectors == VLIB_FRAME_SIZE)
- {
- vlib_put_frame_to_node (wrk->vm, next_index, f);
- wrk->tx_frames[!is_ip4] = 0;
- }
+ st = session_type_from_proto_and_ip (TRANSPORT_PROTO_TCP, is_ip4);
+ session_add_pending_tx_buffer (st, wrk->vm->thread_index, bi);
}
-static void
-tcp_enqueue_to_output (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
- u8 is_ip4)
-{
- tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 0);
-}
-
-static void
-tcp_enqueue_to_output_now (tcp_worker_ctx_t * wrk, vlib_buffer_t * b, u32 bi,
- u8 is_ip4)
-{
- tcp_enqueue_to_output_i (wrk, b, bi, is_ip4, 1);
-}
#endif /* CLIB_MARCH_VARIANT */
static int
int bogus = ~0;
ASSERT ((pkt_ih6->ip_version_traffic_class_and_flow_label & 0xF0) ==
0x60);
- ih6 = vlib_buffer_push_ip6 (vm, b, &pkt_ih6->dst_address,
- &pkt_ih6->src_address, IP_PROTOCOL_TCP);
+ ih6 = vlib_buffer_push_ip6_custom (vm, b, &pkt_ih6->dst_address,
+ &pkt_ih6->src_address,
+ IP_PROTOCOL_TCP,
+ tc->ipv6_flow_label);
th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih6, &bogus);
ASSERT (!bogus);
}
ip6_header_t *ih;
int bogus = ~0;
- ih = vlib_buffer_push_ip6 (vm, b, &tc->c_lcl_ip6,
- &tc->c_rmt_ip6, IP_PROTOCOL_TCP);
+ ih = vlib_buffer_push_ip6_custom (vm, b, &tc->c_lcl_ip6,
+ &tc->c_rmt_ip6, IP_PROTOCOL_TCP,
+ tc->ipv6_flow_label);
th->checksum = ip6_tcp_udp_icmp_compute_checksum (vm, b, ih, &bogus);
ASSERT (!bogus);
}
TCP_EVT (TCP_EVT_SYNACK_SENT, tc);
}
-/**
- * Flush tx frame populated by retransmits and timer pops
- */
-void
-tcp_flush_frame_to_output (tcp_worker_ctx_t * wrk, u8 is_ip4)
-{
- if (wrk->tx_frames[!is_ip4])
- {
- u32 next_index;
- next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
- vlib_put_frame_to_node (wrk->vm, next_index, wrk->tx_frames[!is_ip4]);
- wrk->tx_frames[!is_ip4] = 0;
- }
-}
-
/**
* Flush ip lookup tx frames populated by timer pops
*/
void
tcp_flush_frames_to_output (tcp_worker_ctx_t * wrk)
{
- tcp_flush_frame_to_output (wrk, 1);
- tcp_flush_frame_to_output (wrk, 0);
tcp_flush_frame_to_ip_lookup (wrk, 1);
tcp_flush_frame_to_ip_lookup (wrk, 0);
}
return;
}
+ /* If we have non-dupacks programmed, no need to send them */
+ if ((tc->flags & TCP_CONN_SNDACK) && !tc->pending_dupacks)
+ tc->flags &= ~TCP_CONN_SNDACK;
+
tcp_retransmit_timer_force_update (tc);
b = vlib_get_buffer (vm, bi);
tcp_init_buffer (vm, b);
tcp_make_fin (tc, b);
- tcp_enqueue_to_output_now (wrk, b, bi, tc->c_is_ip4);
+ tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
TCP_EVT (TCP_EVT_FIN_SENT, tc);
/* Account for the FIN */
tc->snd_nxt += 1;
tc->bytes_out += data_len;
tc->data_segs_out += 1;
-
th->checksum = tcp_compute_checksum (tc, b);
TCP_EVT (TCP_EVT_PKTIZE, tc);
}
+always_inline u32
+tcp_buffer_len (vlib_buffer_t * b)
+{
+ u32 data_len = b->current_length;
+ if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
+ data_len += b->total_length_not_including_first_buffer;
+ return data_len;
+}
+
u32
tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
{
tcp_connection_t *tc = (tcp_connection_t *) tconn;
- if (tc->flags & TCP_CONN_TRACK_BURST)
- {
- tcp_bt_check_app_limited (tc);
- tcp_bt_track_tx (tc);
- tc->flags &= ~TCP_CONN_TRACK_BURST;
- }
+ if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
+ tcp_bt_track_tx (tc, tcp_buffer_len (b));
tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0, /* burst */ 1,
/* update_snd_nxt */ 1);
max_deq_bytes = clib_min (available_bytes, max_deq_bytes);
start = tc->snd_una + offset;
+ ASSERT (seq_leq (start + max_deq_bytes, tc->snd_nxt));
+
n_bytes = tcp_prepare_segment (wrk, tc, offset, max_deq_bytes, b);
if (!n_bytes)
return 0;
tcp_send_reset (tc);
tcp_connection_set_state (tc, TCP_STATE_CLOSED);
session_transport_closing_notify (&tc->connection);
+ session_transport_closed_notify (&tc->connection);
tcp_connection_timers_reset (tc);
tcp_timer_update (tc, TCP_TIMER_WAITCLOSE, tcp_cfg.closewait_time);
return;
/* Send the first unacked segment. If we're short on buffers, return
* as soon as possible */
- n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss, &b);
+ n_bytes = clib_min (tc->snd_mss, tc->snd_nxt - tc->snd_una);
+ n_bytes = tcp_prepare_retransmit_segment (wrk, tc, 0, n_bytes, &b);
if (!n_bytes)
{
tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
{
tcp_bt_check_app_limited (tc);
- tcp_bt_track_tx (tc);
+ tcp_bt_track_tx (tc, n_bytes);
}
tcp_push_hdr_i (tc, b, tc->snd_nxt, /* compute opts */ 0,
available_wnd = tc->snd_wnd - offset;
burst_size = clib_min (burst_size, available_wnd / tc->snd_mss);
+ if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
+ tcp_bt_check_app_limited (tc);
+
while (n_segs < burst_size)
{
n_written = tcp_prepare_segment (wrk, tc, offset, tc->snd_mss, &b);
offset += n_written;
n_segs += 1;
+ if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
+ tcp_bt_track_tx (tc, n_written);
+
tc->snd_nxt += n_written;
tc->snd_una_max = seq_max (tc->snd_nxt, tc->snd_una_max);
}
u32 tx_adv_sack = sb->high_sacked - tc->snd_congestion;
f64 rr = (f64) tc->ssthresh / tc->prev_cwnd;
+ if (tcp_fastrecovery_first (tc))
+ return 1;
+
return (tx_adv_sack > (tc->snd_una - tc->prr_start) * rr);
}
+static inline u8
+tcp_max_tx_deq (tcp_connection_t * tc)
+{
+ return (transport_max_tx_dequeue (&tc->connection)
+ - (tc->snd_nxt - tc->snd_una));
+}
+
#define scoreboard_rescue_rxt_valid(_sb, _tc) \
(seq_geq (_sb->rescue_rxt, _tc->snd_una) \
&& seq_leq (_sb->rescue_rxt, _tc->snd_congestion))
vlib_buffer_t *b = 0;
sack_scoreboard_t *sb;
int snd_space;
- u64 time_now;
ASSERT (tcp_in_cong_recovery (tc));
- time_now = wrk->vm->clib_time.last_cpu_time;
- burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
- time_now);
+ burst_bytes = transport_connection_tx_pacer_burst (&tc->connection);
burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss);
if (!burst_size)
{
sb = &tc->sack_sb;
/* Check if snd_una is a lost retransmit */
- if (seq_gt (sb->high_sacked, tc->snd_congestion)
+ if (pool_elts (sb->holes)
+ && seq_gt (sb->high_sacked, tc->snd_congestion)
&& tc->rxt_head != tc->snd_una
&& tcp_retransmit_should_retry_head (tc, sb))
{
- n_written = tcp_prepare_retransmit_segment (wrk, tc, 0, tc->snd_mss,
- &b);
+ max_bytes = clib_min (tc->snd_mss, tc->snd_congestion - tc->snd_una);
+ n_written = tcp_prepare_retransmit_segment (wrk, tc, 0, max_bytes, &b);
if (!n_written)
{
tcp_program_retransmit (tc);
ASSERT (tc->rxt_delivered <= tc->snd_rxt_bytes);
}
+ tcp_fastrecovery_first_off (tc);
+
TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
hole = scoreboard_get_hole (sb, sb->cur_rxt_hole);
while (snd_space > 0 && n_segs < burst_size)
{
- hole = scoreboard_next_rxt_hole (sb, hole, max_deq, &can_rescue,
+ hole = scoreboard_next_rxt_hole (sb, hole, max_deq != 0, &can_rescue,
&snd_limited);
if (!hole)
{
tcp_enqueue_to_output (wrk, b, bi, tc->c_is_ip4);
sb->high_rxt += n_written;
+ ASSERT (seq_leq (sb->high_rxt, tc->snd_nxt));
+
snd_space -= n_written;
n_segs += 1;
}
if (reset_pacer)
{
- transport_connection_tx_pacer_reset_bucket (&tc->connection,
- vm->clib_time.
- last_cpu_time);
+ transport_connection_tx_pacer_reset_bucket (&tc->connection);
}
else
{
tcp_retransmit_no_sack (tcp_worker_ctx_t * wrk, tcp_connection_t * tc,
u32 burst_size)
{
- u32 n_written = 0, offset = 0, bi, max_deq, n_segs_now;
+ u32 n_written = 0, offset = 0, bi, max_deq, n_segs_now, max_bytes;
u32 burst_bytes, sent_bytes;
vlib_main_t *vm = wrk->vm;
int snd_space, n_segs = 0;
u8 cc_limited = 0;
vlib_buffer_t *b;
- u64 time_now;
- ASSERT (tcp_in_fastrecovery (tc));
+ ASSERT (tcp_in_cong_recovery (tc));
TCP_EVT (TCP_EVT_CC_EVT, tc, 0);
- time_now = wrk->vm->clib_time.last_cpu_time;
- burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
- time_now);
+ burst_bytes = transport_connection_tx_pacer_burst (&tc->connection);
burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss);
if (!burst_size)
{
* segment. */
while (snd_space > 0 && n_segs < burst_size)
{
- n_written = tcp_prepare_retransmit_segment (wrk, tc, offset,
- tc->snd_mss, &b);
+ max_bytes = clib_min (tc->snd_mss,
+ tc->snd_congestion - tc->snd_una - offset);
+ if (!max_bytes)
+ break;
+ n_written = tcp_prepare_retransmit_segment (wrk, tc, offset, max_bytes,
+ &b);
/* Nothing left to retransmit */
if (n_written == 0)
snd_space = clib_min (max_deq, snd_space);
burst_size = clib_min (burst_size - n_segs, snd_space / tc->snd_mss);
n_segs_now = tcp_transmit_unsent (wrk, tc, burst_size);
- if (max_deq > n_segs_now * tc->snd_mss)
+ if (n_segs_now && max_deq > n_segs_now * tc->snd_mss)
tcp_program_retransmit (tc);
n_segs += n_segs_now;
}
if (!tc->pending_dupacks)
{
- tcp_send_ack (tc);
- return 1;
+ if (tcp_in_cong_recovery (tc) || !tcp_max_tx_deq (tc)
+ || tc->state != TCP_STATE_ESTABLISHED)
+ {
+ tcp_send_ack (tc);
+ return 1;
+ }
+ return 0;
}
/* If we're supposed to send dupacks but have no ooo data
if (!vec_len (tc->snd_sacks))
{
tcp_send_ack (tc);
+ tc->pending_dupacks = 0;
return 1;
}
tcp_worker_ctx_t *wrk;
u32 n_segs;
+ if (PREDICT_FALSE (tc->state == TCP_STATE_CLOSED))
+ return 0;
+
wrk = tcp_get_worker (tc->c_thread_index);
if (tcp_opts_sack_permitted (&tc->rcv_opts))
ih0 = vlib_buffer_push_ip4 (vm, b0, &tc0->c_lcl_ip4, &tc0->c_rmt_ip4,
IP_PROTOCOL_TCP, tcp_csum_offload (tc0));
else
- ih0 = vlib_buffer_push_ip6 (vm, b0, &tc0->c_lcl_ip6, &tc0->c_rmt_ip6,
- IP_PROTOCOL_TCP);
+ ih0 =
+ vlib_buffer_push_ip6_custom (vm, b0, &tc0->c_lcl_ip6, &tc0->c_rmt_ip6,
+ IP_PROTOCOL_TCP, tc0->ipv6_flow_label);
}
Code Review / vpp.git / blobdiff