if (tcp_in_cong_recovery (tc))
{
/* Accept rtt estimates for samples that have not been retransmitted */
- if ((tc->flags & TCP_CONN_RATE_SAMPLE) && !(rs->flags & TCP_BTS_IS_RXT))
+ if ((tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
+ && !(rs->flags & TCP_BTS_IS_RXT))
{
mrtt = rs->rtt_time * THZ;
goto estimate_rtt;
tcp_update_rto (tc);
}
+always_inline u8
+tcp_recovery_no_snd_space (tcp_connection_t * tc)
+{
+ return (tcp_in_fastrecovery (tc)
+ && tcp_fastrecovery_prr_snd_space (tc) < tc->snd_mss)
+ || (tcp_in_recovery (tc)
+ && tcp_available_output_snd_space (tc) < tc->snd_mss);
+}
+
/**
* Dequeue bytes for connections that have received acks in last burst
*/
tc = tcp_connection_get (pending_deq_acked[i], thread_index);
tc->flags &= ~TCP_CONN_DEQ_PENDING;
- if (PREDICT_FALSE (!tc->burst_acked))
- continue;
+ if (tc->burst_acked)
+ {
+ /* Dequeue the newly ACKed bytes */
+ session_tx_fifo_dequeue_drop (&tc->connection, tc->burst_acked);
+ tc->burst_acked = 0;
+ tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
- /* Dequeue the newly ACKed bytes */
- session_tx_fifo_dequeue_drop (&tc->connection, tc->burst_acked);
- tc->burst_acked = 0;
- tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
+ if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
+ {
+ if (seq_leq (tc->psh_seq, tc->snd_una))
+ tc->flags &= ~TCP_CONN_PSH_PENDING;
+ }
- if (PREDICT_FALSE (tc->flags & TCP_CONN_PSH_PENDING))
- {
- if (seq_leq (tc->psh_seq, tc->snd_una))
- tc->flags &= ~TCP_CONN_PSH_PENDING;
- }
+ /* If everything has been acked, stop retransmit timer
+ * otherwise update. */
+ tcp_retransmit_timer_update (tc);
- /* If everything has been acked, stop retransmit timer
- * otherwise update. */
- tcp_retransmit_timer_update (tc);
+ /* Update pacer based on our new cwnd estimate */
+ tcp_connection_tx_pacer_update (tc);
+ }
- /* Update pacer based on our new cwnd estimate */
- tcp_connection_tx_pacer_update (tc);
+ /* Reset the pacer if we've been idle, i.e., no data sent or if
+ * we're in recovery and snd space constrained */
+ if (tc->data_segs_out == tc->prev_dsegs_out
+ || tcp_recovery_no_snd_space (tc))
+ transport_connection_tx_pacer_reset_bucket (&tc->connection,
+ wrk->vm->clib_time.
+ last_cpu_time);
+ tc->prev_dsegs_out = tc->data_segs_out;
}
_vec_len (wrk->pending_deq_acked) = 0;
}
tc->burst_acked += tc->bytes_acked;
}
-/**
- * Check if duplicate ack as per RFC5681 Sec. 2
- */
-static u8
-tcp_ack_is_dupack (tcp_connection_t * tc, vlib_buffer_t * b, u32 prev_snd_wnd,
- u32 prev_snd_una)
-{
- return ((vnet_buffer (b)->tcp.ack_number == prev_snd_una)
- && seq_gt (tc->snd_nxt, tc->snd_una)
- && (vnet_buffer (b)->tcp.seq_end == vnet_buffer (b)->tcp.seq_number)
- && (prev_snd_wnd == tc->snd_wnd));
-}
-
-/**
- * Checks if ack is a congestion control event.
- */
-static u8
-tcp_ack_is_cc_event (tcp_connection_t * tc, vlib_buffer_t * b,
- u32 prev_snd_wnd, u32 prev_snd_una, u8 * is_dack)
-{
- /* Check if ack is duplicate. Per RFC 6675, ACKs that SACK new data are
- * defined to be 'duplicate' */
- *is_dack = tc->sack_sb.last_sacked_bytes
- || tcp_ack_is_dupack (tc, b, prev_snd_wnd, prev_snd_una);
-
- return ((*is_dack || tcp_in_cong_recovery (tc)) && !tcp_is_lost_fin (tc));
-}
-
#ifndef CLIB_MARCH_VARIANT
static u32
scoreboard_hole_index (sack_scoreboard_t * sb, sack_scoreboard_hole_t * hole)
}
void
-scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 snd_una)
+scoreboard_init_rxt (sack_scoreboard_t * sb, u32 snd_una)
{
sack_scoreboard_hole_t *hole;
hole = scoreboard_first_hole (sb);
sb->sacked_bytes = 0;
sb->last_sacked_bytes = 0;
sb->last_bytes_delivered = 0;
- sb->high_sacked = 0;
- sb->high_rxt = 0;
sb->lost_bytes = 0;
sb->last_lost_bytes = 0;
sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
last_hole->is_lost = 1;
sb->tail = scoreboard_hole_index (sb, last_hole);
sb->high_sacked = start;
- scoreboard_init_high_rxt (sb, start);
+ scoreboard_init_rxt (sb, start);
}
#endif /* CLIB_MARCH_VARIANT */
ASSERT (sb->last_lost_bytes <= sb->lost_bytes);
ASSERT ((ack - tc->snd_una) + sb->last_sacked_bytes
- sb->last_bytes_delivered >= sb->rxt_sacked);
+ ASSERT ((ack - tc->snd_una) >= tc->sack_sb.last_bytes_delivered
+ || (tc->flags & TCP_CONN_FINSNT));
TCP_EVT (TCP_EVT_CC_SCOREBOARD, tc);
}
tc->snd_rxt_bytes = 0;
tc->rxt_delivered = 0;
tc->prr_delivered = 0;
+ tc->prr_start = tc->snd_una;
tc->prev_ssthresh = tc->ssthresh;
tc->prev_cwnd = tc->cwnd;
+ tc->snd_rxt_ts = tcp_tstamp (tc);
tcp_cc_congestion (tc);
/* Post retransmit update cwnd to ssthresh and account for the
}
static inline u8
-tcp_should_fastrecover (tcp_connection_t * tc)
+tcp_should_fastrecover (tcp_connection_t * tc, u8 has_sack)
{
+ if (!has_sack)
+ {
+ /* If of of the two conditions lower hold, reset dupacks because
+ * we're probably after timeout (RFC6582 heuristics).
+ * If Cumulative ack does not cover more than congestion threshold,
+ * and:
+ * 1) The following doesn't hold: The congestion window is greater
+ * than SMSS bytes and the difference between highest_ack
+ * and prev_highest_ack is at most 4*SMSS bytes
+ * 2) Echoed timestamp in the last non-dup ack does not equal the
+ * stored timestamp
+ */
+ if (seq_leq (tc->snd_una, tc->snd_congestion)
+ && ((!(tc->cwnd > tc->snd_mss
+ && tc->bytes_acked <= 4 * tc->snd_mss))
+ || (tc->rcv_opts.tsecr != tc->tsecr_last_ack)))
+ {
+ tc->rcv_dupacks = 0;
+ return 0;
+ }
+ }
return ((tc->rcv_dupacks == TCP_DUPACK_THRESHOLD)
|| tcp_should_fastrecover_sack (tc));
}
is_spurious = 1;
}
+ tc->rcv_dupacks = 0;
tc->prr_delivered = 0;
tc->rxt_delivered = 0;
tc->snd_rxt_bytes = 0;
tc->rtt_ts = 0;
tc->flags &= ~TCP_CONN_RXT_PENDING;
+ tcp_connection_tx_pacer_reset (tc, tc->cwnd, 0 /* start bucket */ );
+
/* Previous recovery left us congested. Continue sending as part
* of the current recovery event with an updated snd_congestion */
if (tc->sack_sb.sacked_bytes)
{
tc->snd_congestion = tc->snd_nxt;
- if (tcp_opts_sack_permitted (&tc->rcv_opts))
- scoreboard_init_high_rxt (&tc->sack_sb, tc->snd_una);
+ tc->snd_rxt_ts = tcp_tstamp (tc);
+ tc->prr_start = tc->snd_una;
+ scoreboard_init_rxt (&tc->sack_sb, tc->snd_una);
tcp_program_retransmit (tc);
return is_spurious;
}
tc->snd_congestion = tc->snd_una - 1;
}
-static void
-tcp_update_delivered (tcp_connection_t * tc, u8 is_dack, u8 has_sack)
-{
- if (has_sack)
- {
- ASSERT (tc->bytes_acked >= tc->sack_sb.last_bytes_delivered
- || (tc->flags & TCP_CONN_FINSNT));
-
- tc->rxt_delivered += tc->sack_sb.rxt_sacked;
- tc->prr_delivered += tc->bytes_acked + tc->sack_sb.last_sacked_bytes
- - tc->sack_sb.last_bytes_delivered;
- }
- else
- {
- tcp_fastrecovery_first_on (tc);
- tc->rxt_delivered = clib_max (tc->rxt_delivered + tc->bytes_acked,
- tc->snd_rxt_bytes);
- if (is_dack)
- tc->prr_delivered += is_dack;
- else
- tc->prr_delivered += tc->bytes_acked - tc->snd_mss * tc->rcv_dupacks;
- }
-}
-
/**
* One function to rule them all ... and in the darkness bind them
*/
u8 has_sack = tcp_opts_sack_permitted (&tc->rcv_opts);
/*
- * Already in fast recovery. Return if no data acked, partial acks
- * and accounting for segments that have left the network are done
- * lower.
+ * If not in recovery, figure out if we should enter
*/
- if (tcp_in_cong_recovery (tc))
+ if (!tcp_in_cong_recovery (tc))
{
- if (!has_sack)
- tc->rcv_dupacks += is_dack;
-
- if (!tc->bytes_acked)
- {
- tcp_update_delivered (tc, is_dack, has_sack);
- tcp_program_retransmit (tc);
- if (!has_sack)
- tcp_cc_rcv_cong_ack (tc, TCP_CC_DUPACK, rs);
- return;
- }
- }
- /*
- * Duplicate ACK. Check if we should enter fast recovery
- */
- else if (is_dack)
- {
- TCP_EVT (TCP_EVT_DUPACK_RCVD, tc, 1);
- ASSERT (tc->snd_una != tc->snd_nxt || tc->sack_sb.last_sacked_bytes);
-
- /* Heuristic to catch potential late dupacks */
- if (!tc->sack_sb.sacked_bytes && tc->snd_una == tc->snd_congestion
- && timestamp_leq (tc->rcv_opts.tsecr, tc->tsecr_last_ack))
- return;
+ ASSERT (is_dack);
tc->rcv_dupacks++;
+ TCP_EVT (TCP_EVT_DUPACK_RCVD, tc, 1);
+ tcp_cc_rcv_cong_ack (tc, TCP_CC_DUPACK, rs);
- if (tcp_should_fastrecover (tc))
+ if (tcp_should_fastrecover (tc, has_sack))
{
- ASSERT (!tcp_in_fastrecovery (tc));
-
tcp_cc_init_congestion (tc);
if (has_sack)
- scoreboard_init_high_rxt (&tc->sack_sb, tc->snd_una);
+ scoreboard_init_rxt (&tc->sack_sb, tc->snd_una);
tcp_connection_tx_pacer_reset (tc, tc->cwnd, 0 /* start bucket */ );
tcp_program_retransmit (tc);
- return;
}
- else
- {
- tcp_cc_rcv_cong_ack (tc, TCP_CC_DUPACK, rs);
- return;
- }
- }
- /* Don't allow entry in fast recovery if still in recovery, for now */
- else if (0 && is_dack && tcp_in_recovery (tc))
- {
- /* If of of the two conditions lower hold, reset dupacks because
- * we're probably after timeout (RFC6582 heuristics).
- * If Cumulative ack does not cover more than congestion threshold,
- * and:
- * 1) The following doesn't hold: The congestion window is greater
- * than SMSS bytes and the difference between highest_ack
- * and prev_highest_ack is at most 4*SMSS bytes
- * 2) Echoed timestamp in the last non-dup ack does not equal the
- * stored timestamp
- */
- if (seq_leq (tc->snd_una, tc->snd_congestion)
- && ((!(tc->cwnd > tc->snd_mss
- && tc->bytes_acked <= 4 * tc->snd_mss))
- || (tc->rcv_opts.tsecr != tc->tsecr_last_ack)))
- {
- tc->rcv_dupacks = 0;
- return;
- }
- }
- ASSERT (tc->bytes_acked);
- TCP_EVT (TCP_EVT_CC_PACK, tc);
+ return;
+ }
/*
- * Legitimate ACK. 1) See if we can exit recovery
+ * Already in recovery. See if we can exit and stop retransmitting
*/
- /* RFC6675: If the incoming ACK is a cumulative acknowledgment,
- * reset dupacks to 0. Also needed if in congestion recovery */
- tc->rcv_dupacks = 0;
-
if (seq_geq (tc->snd_una, tc->snd_congestion))
{
/* If spurious return, we've already updated everything */
}
/*
- * Legitimate ACK. 2) If PARTIAL ACK try to retransmit
+ * Process (re)transmit feedback. Output path uses this to decide how much
+ * more data to release into the network
+ */
+ if (has_sack)
+ {
+ tc->rxt_delivered += tc->sack_sb.rxt_sacked;
+ tc->prr_delivered += tc->bytes_acked + tc->sack_sb.last_sacked_bytes
+ - tc->sack_sb.last_bytes_delivered;
+
+ tcp_program_retransmit (tc);
+ }
+ else
+ {
+ if (is_dack)
+ {
+ tc->rcv_dupacks += 1;
+ TCP_EVT (TCP_EVT_DUPACK_RCVD, tc, 1);
+ }
+ tc->rxt_delivered = clib_max (tc->rxt_delivered + tc->bytes_acked,
+ tc->snd_rxt_bytes);
+ if (is_dack)
+ tc->prr_delivered += 1;
+ else
+ tc->prr_delivered += tc->bytes_acked - tc->snd_mss * tc->rcv_dupacks;
+
+ /* If partial ack, assume that the first un-acked segment was lost */
+ if (tc->bytes_acked || tc->rcv_dupacks == TCP_DUPACK_THRESHOLD)
+ tcp_fastrecovery_first_on (tc);
+
+ tcp_program_retransmit (tc);
+ }
+
+ /*
+ * Notify cc of the event
*/
- /* Remove retransmitted bytes that have been delivered */
- tcp_update_delivered (tc, is_dack, has_sack);
+ if (!tc->bytes_acked)
+ {
+ tcp_cc_rcv_cong_ack (tc, TCP_CC_DUPACK, rs);
+ return;
+ }
+
+ /* RFC6675: If the incoming ACK is a cumulative acknowledgment,
+ * reset dupacks to 0. Also needed if in congestion recovery */
+ tc->rcv_dupacks = 0;
if (tcp_in_recovery (tc))
tcp_cc_rcv_ack (tc, rs);
else
tcp_cc_rcv_cong_ack (tc, TCP_CC_PARTIALACK, rs);
+}
- /*
- * Since this was a partial ack, try to retransmit some more data
- */
- tcp_program_retransmit (tc);
+/**
+ * Check if duplicate ack as per RFC5681 Sec. 2
+ */
+always_inline u8
+tcp_ack_is_dupack (tcp_connection_t * tc, vlib_buffer_t * b, u32 prev_snd_wnd,
+ u32 prev_snd_una)
+{
+ return ((vnet_buffer (b)->tcp.ack_number == prev_snd_una)
+ && seq_gt (tc->snd_nxt, tc->snd_una)
+ && (vnet_buffer (b)->tcp.seq_end == vnet_buffer (b)->tcp.seq_number)
+ && (prev_snd_wnd == tc->snd_wnd));
+}
+
+/**
+ * Checks if ack is a congestion control event.
+ */
+static u8
+tcp_ack_is_cc_event (tcp_connection_t * tc, vlib_buffer_t * b,
+ u32 prev_snd_wnd, u32 prev_snd_una, u8 * is_dack)
+{
+ /* Check if ack is duplicate. Per RFC 6675, ACKs that SACK new data are
+ * defined to be 'duplicate' as well */
+ *is_dack = tc->sack_sb.last_sacked_bytes
+ || tcp_ack_is_dupack (tc, b, prev_snd_wnd, prev_snd_una);
+
+ /* If reneging, wait for timer based retransmits */
+ if (PREDICT_FALSE (tcp_is_lost_fin (tc) || tc->sack_sb.is_reneging))
+ return 0;
+
+ return (*is_dack || tcp_in_cong_recovery (tc));
}
/**
tc->snd_una = vnet_buffer (b)->tcp.ack_number;
tcp_validate_txf_size (tc, tc->bytes_acked);
- if (tc->flags & TCP_CONN_RATE_SAMPLE)
+ if (tc->cfg_flags & TCP_CFG_F_RATE_SAMPLE)
tcp_bt_sample_delivery_rate (tc, &rs);
+ tcp_program_dequeue (wrk, tc);
+
if (tc->bytes_acked)
- {
- tcp_program_dequeue (wrk, tc);
- tcp_update_rtt (tc, &rs, vnet_buffer (b)->tcp.ack_number);
- }
+ tcp_update_rtt (tc, &rs, vnet_buffer (b)->tcp.ack_number);
TCP_EVT (TCP_EVT_ACK_RCVD, tc);
sw_if_idx = dpo->dpoi_index;
hw_if = vnet_get_sup_hw_interface (vnm, sw_if_idx);
- tc->is_tso =
- ((hw_if->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO) == 0) ? 0 : 1;
+ if (hw_if->flags & VNET_HW_INTERFACE_FLAG_SUPPORTS_GSO)
+ tc->cfg_flags |= TCP_CFG_F_TSO;
}
-
always_inline uword
tcp46_syn_sent_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
vlib_frame_t * from_frame, int is_ip4)
goto drop;
}
- tcp_check_tx_offload (new_tc0, is_ip4);
+ if (!(new_tc0->cfg_flags & TCP_CFG_F_NO_TSO))
+ tcp_check_tx_offload (new_tc0, is_ip4);
/* Read data, if any */
if (PREDICT_FALSE (vnet_buffer (b0)->tcp.data_len))
tc0->state = TCP_STATE_ESTABLISHED;
TCP_EVT (TCP_EVT_STATE_CHANGE, tc0);
- tcp_check_tx_offload (tc0, is_ip4);
+ if (!(tc0->cfg_flags & TCP_CFG_F_NO_TSO))
+ tcp_check_tx_offload (tc0, is_ip4);
/* Initialize session variables */
tc0->snd_una = vnet_buffer (b0)->tcp.ack_number;
Code Review / vpp.git / blobdiff