{
ASSERT (timestamp_leq (tc->tsval_recent, tc->rcv_opts.tsval));
tc->tsval_recent = tc->rcv_opts.tsval;
- tc->tsval_recent_age = tcp_time_now ();
+ tc->tsval_recent_age = tcp_time_now_w_thread (tc->c_thread_index);
}
}
/* If it just so happens that a segment updates tsval_recent for a
* segment over 24 days old, invalidate tsval_recent. */
if (timestamp_lt (tc0->tsval_recent_age + TCP_PAWS_IDLE,
- tcp_time_now ()))
+ tcp_time_now_w_thread (tc0->c_thread_index)))
{
/* Age isn't reset until we get a valid tsval (bsd inspired) */
tc0->tsval_recent = 0;
if (tc->rtt_ts && seq_geq (ack, tc->rtt_seq))
{
- mrtt = tcp_time_now () - tc->rtt_ts;
+ tc->mrtt_us = tcp_time_now_us (tc->c_thread_index) - tc->rtt_ts;
+ mrtt = clib_max ((u32) (tc->mrtt_us * THZ), 1);
}
/* As per RFC7323 TSecr can be used for RTTM only if the segment advances
* snd_una, i.e., the left side of the send window:
* seq_lt (tc->snd_una, ack). This is a condition for calling update_rtt */
else if (tcp_opts_tstamp (&tc->rcv_opts) && tc->rcv_opts.tsecr)
{
- mrtt = tcp_time_now () - tc->rcv_opts.tsecr;
+ u32 now = tcp_time_now_w_thread (tc->c_thread_index);
+ mrtt = clib_max (now - tc->rcv_opts.tsecr, 1);
}
/* Ignore dubious measurements */
sack_scoreboard_hole_t *
scoreboard_next_rxt_hole (sack_scoreboard_t * sb,
sack_scoreboard_hole_t * start,
- u8 have_sent_1_smss,
- u8 * can_rescue, u8 * snd_limited)
+ u8 have_unsent, u8 * can_rescue, u8 * snd_limited)
{
sack_scoreboard_hole_t *hole = 0;
}
else
{
- /* Rule (2): output takes care of transmitting new data */
- if (!have_sent_1_smss)
+ /* Rule (2): available unsent data */
+ if (have_unsent)
{
- hole = 0;
sb->cur_rxt_hole = TCP_INVALID_SACK_HOLE_INDEX;
+ return 0;
}
/* Rule (3): if hole not lost */
else if (seq_lt (hole->start, sb->high_sacked))
}
static void
-scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 seq)
+scoreboard_init_high_rxt (sack_scoreboard_t * sb, u32 snd_una)
{
sack_scoreboard_hole_t *hole;
hole = scoreboard_first_hole (sb);
if (hole)
{
- seq = seq_gt (seq, hole->start) ? seq : hole->start;
+ snd_una = seq_gt (snd_una, hole->start) ? snd_una : hole->start;
sb->cur_rxt_hole = sb->head;
}
- sb->high_rxt = seq;
+ sb->high_rxt = snd_una;
+ sb->rescue_rxt = snd_una - 1;
}
void
tc->snd_nxt = tc->snd_una_max;
tc->snd_rxt_bytes = 0;
- /* HACK: since we don't have an output pacer, force slow start */
- tc->cwnd = 20 * tc->snd_mss;
-
tcp_fastrecovery_off (tc);
tcp_fastrecovery_1_smss_off (tc);
tcp_fastrecovery_first_off (tc);
+
TCP_EVT_DBG (TCP_EVT_CC_EVT, tc, 3);
}
ASSERT (!tcp_in_cong_recovery (tc) || tcp_is_lost_fin (tc));
/* Congestion avoidance */
- tc->cc_algo->rcv_ack (tc);
- tc->tsecr_last_ack = tc->rcv_opts.tsecr;
+ tcp_cc_rcv_ack (tc);
/* If a cumulative ack, make sure dupacks is 0 */
tc->rcv_dupacks = 0;
}
void
-tcp_program_fastretransmit (tcp_connection_t * tc)
+tcp_program_fastretransmit (tcp_worker_ctx_t * wrk, tcp_connection_t * tc)
{
- tcp_worker_ctx_t *wrk = &tcp_main.wrk_ctx[tc->c_thread_index];
if (!(tc->flags & TCP_CONN_FRXT_PENDING))
{
vec_add1 (wrk->pending_fast_rxt, tc->c_c_index);
}
void
-tcp_do_fastretransmits (u32 thread_index)
+tcp_do_fastretransmits (tcp_worker_ctx_t * wrk)
{
- tcp_worker_ctx_t *wrk = &tcp_main.wrk_ctx[thread_index];
- u32 max_burst_size, burst_size, n_segs = 0;
+ u32 *ongoing_fast_rxt, burst_bytes, sent_bytes, thread_index;
+ u32 max_burst_size, burst_size, n_segs = 0, n_segs_now;
tcp_connection_t *tc;
+ u64 last_cpu_time;
int i;
- if (vec_len (wrk->pending_fast_rxt) == 0)
+ if (vec_len (wrk->pending_fast_rxt) == 0
+ && vec_len (wrk->postponed_fast_rxt) == 0)
return;
- vec_append (wrk->ongoing_fast_rxt, wrk->pending_fast_rxt);
- vec_reset_length (wrk->pending_fast_rxt);
+ thread_index = wrk->vm->thread_index;
+ last_cpu_time = wrk->vm->clib_time.last_cpu_time;
+ ongoing_fast_rxt = wrk->ongoing_fast_rxt;
+ vec_append (ongoing_fast_rxt, wrk->postponed_fast_rxt);
+ vec_append (ongoing_fast_rxt, wrk->pending_fast_rxt);
- max_burst_size = VLIB_FRAME_SIZE / vec_len (wrk->ongoing_fast_rxt);
+ _vec_len (wrk->postponed_fast_rxt) = 0;
+ _vec_len (wrk->pending_fast_rxt) = 0;
+
+ max_burst_size = VLIB_FRAME_SIZE / vec_len (ongoing_fast_rxt);
max_burst_size = clib_max (max_burst_size, 1);
- for (i = 0; i < vec_len (wrk->ongoing_fast_rxt); i++)
+ for (i = 0; i < vec_len (ongoing_fast_rxt); i++)
{
- tc = tcp_connection_get (wrk->ongoing_fast_rxt[i], thread_index);
+ if (n_segs >= VLIB_FRAME_SIZE)
+ {
+ vec_add1 (wrk->postponed_fast_rxt, ongoing_fast_rxt[i]);
+ continue;
+ }
+
+ tc = tcp_connection_get (ongoing_fast_rxt[i], thread_index);
tc->flags &= ~TCP_CONN_FRXT_PENDING;
if (!tcp_in_fastrecovery (tc))
continue;
- /* TODO tx pacer instead of this */
- if (n_segs >= VLIB_FRAME_SIZE)
+ burst_size = clib_min (max_burst_size, VLIB_FRAME_SIZE - n_segs);
+ burst_bytes = transport_connection_tx_pacer_burst (&tc->connection,
+ last_cpu_time);
+ burst_size = clib_min (burst_size, burst_bytes / tc->snd_mss);
+ if (!burst_size)
{
- tcp_program_fastretransmit (tc);
+ tcp_program_fastretransmit (wrk, tc);
continue;
}
- burst_size = clib_min (max_burst_size, VLIB_FRAME_SIZE - n_segs);
-
- if (tc->cwnd > tc->ssthresh + 3 * tc->snd_mss)
- {
- /* The first segment MUST be retransmitted */
- if (tcp_retransmit_first_unacked (tc))
- {
- tcp_program_fastretransmit (tc);
- continue;
- }
-
- /* Post retransmit update cwnd to ssthresh and account for the
- * three segments that have left the network and should've been
- * buffered at the receiver XXX */
- tc->cwnd = tc->ssthresh + 3 * tc->snd_mss;
-
- /* If cwnd allows, send more data */
- if (tcp_opts_sack_permitted (&tc->rcv_opts))
- {
- scoreboard_init_high_rxt (&tc->sack_sb,
- tc->snd_una + tc->snd_mss);
- tc->sack_sb.rescue_rxt = tc->snd_una - 1;
- n_segs += tcp_fast_retransmit_sack (tc, burst_size);
- }
- else
- {
- n_segs += tcp_fast_retransmit_no_sack (tc, burst_size);
- }
- }
- else
- n_segs += tcp_fast_retransmit (tc, burst_size);
+ n_segs_now = tcp_fast_retransmit (wrk, tc, burst_size);
+ sent_bytes = clib_min (n_segs_now * tc->snd_mss, burst_bytes);
+ transport_connection_tx_pacer_update_bytes (&tc->connection,
+ sent_bytes);
+ n_segs += n_segs_now;
}
- vec_reset_length (wrk->ongoing_fast_rxt);
+ _vec_len (ongoing_fast_rxt) = 0;
+ wrk->ongoing_fast_rxt = ongoing_fast_rxt;
}
/**
{
if (tc->bytes_acked)
goto partial_ack;
- tcp_program_fastretransmit (tc);
+ tcp_program_fastretransmit (tcp_get_worker (tc->c_thread_index), tc);
return;
}
/*
}
else if (tcp_should_fastrecover (tc))
{
+ u32 pacer_wnd;
+
ASSERT (!tcp_in_fastrecovery (tc));
/* Heuristic to catch potential late dupacks
tc->cc_algo->rcv_cong_ack (tc, TCP_CC_DUPACK);
if (tcp_opts_sack_permitted (&tc->rcv_opts))
- tc->sack_sb.high_rxt = tc->snd_una;
+ {
+ tc->cwnd = tc->ssthresh;
+ scoreboard_init_high_rxt (&tc->sack_sb, tc->snd_una);
+ }
+ else
+ {
+ /* Post retransmit update cwnd to ssthresh and account for the
+ * three segments that have left the network and should've been
+ * buffered at the receiver XXX */
+ tc->cwnd = tc->ssthresh + 3 * tc->snd_mss;
+ }
- tcp_program_fastretransmit (tc);
+ /* Constrain rate until we get a partial ack */
+ pacer_wnd = clib_max (0.1 * tc->cwnd, 2 * tc->snd_mss);
+ tcp_connection_tx_pacer_reset (tc, pacer_wnd,
+ 0 /* start bucket */ );
+ tcp_program_fastretransmit (tcp_get_worker (tc->c_thread_index),
+ tc);
return;
}
else if (!tc->bytes_acked
tc->snd_nxt = tc->snd_una_max;
/* Treat as congestion avoidance ack */
- tc->cc_algo->rcv_ack (tc);
- tc->tsecr_last_ack = tc->rcv_opts.tsecr;
+ tcp_cc_rcv_ack (tc);
return;
}
* Legitimate ACK. 2) If PARTIAL ACK try to retransmit
*/
+ /* Update the pacing rate. For the first partial ack we move from
+ * the artificially constrained rate to the one after congestion */
+ tcp_connection_tx_pacer_update (tc);
+
/* XXX limit this only to first partial ack? */
tcp_retransmit_timer_force_update (tc);
/* Post RTO timeout don't try anything fancy */
if (tcp_in_recovery (tc))
{
- tc->cc_algo->rcv_ack (tc);
- tc->tsecr_last_ack = tc->rcv_opts.tsecr;
+ tcp_cc_rcv_ack (tc);
transport_add_tx_event (&tc->connection);
return;
}
{
/* Apparently all retransmitted holes have been acked */
tc->snd_rxt_bytes = 0;
+ tc->sack_sb.high_rxt = tc->snd_una;
}
}
else
{
+ tcp_fastrecovery_first_on (tc);
+ /* Reuse last bytes delivered to track total bytes acked */
+ tc->sack_sb.last_bytes_delivered += tc->bytes_acked;
if (tc->snd_rxt_bytes > tc->bytes_acked)
tc->snd_rxt_bytes -= tc->bytes_acked;
else
/*
* Since this was a partial ack, try to retransmit some more data
*/
- tcp_program_fastretransmit (tc);
+ tcp_program_fastretransmit (tcp_get_worker (tc->c_thread_index), tc);
}
/**
{
tcp_make_ack (tc, b);
*next = tcp_next_output (tc->c_is_ip4);
- *error = TCP_ERROR_ACK_INVALID;
+ *error = TCP_ERROR_ACK_FUTURE;
TCP_EVT_DBG (TCP_EVT_ACK_RCV_ERR, tc, 0,
vnet_buffer (b)->tcp.ack_number);
return -1;
vnet_buffer (b)->tcp.ack_number);
tc->snd_nxt = vnet_buffer (b)->tcp.ack_number;
- *error = TCP_ERROR_ACK_FUTURE;
}
/* If old ACK, probably it's an old dupack */
thread_index);
err_counters[TCP_ERROR_EVENT_FIFO_FULL] = errors;
tcp_store_err_counters (established, err_counters);
- tcp_flush_frame_to_output (vm, thread_index, is_ip4);
+ tcp_flush_frame_to_output (tcp_get_worker (thread_index), is_ip4);
return frame->n_vectors;
}
vlib_buffer_t *bufs[VLIB_FRAME_SIZE], **b;
u16 nexts[VLIB_FRAME_SIZE], *next;
- tcp_set_time_now (thread_index);
+ tcp_set_time_now (tcp_get_worker (thread_index));
from = vlib_frame_vector_args (frame);
n_left_from = frame->n_vectors;
Code Review / vpp.git / blobdiff