- cache and reuse tcp options and rcv_wnd for session layer tx bursts
- avoid reading/setting total_length_not_including_first_buffer. It's
part of a buffer's second cache line so it comes at a "cost".
Change-Id: Id18219c2f7e07cf4c63ee74f9cdd9e5918904036
Signed-off-by: Florin Coras <fcoras@cisco.com>
b->error = 0;
b->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED;
b->current_data = 0;
b->error = 0;
b->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED;
b->current_data = 0;
- b->total_length_not_including_first_buffer = 0;
data0 = vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
len_to_deq = clib_min (ctx->left_to_snd, ctx->deq_per_first_buf);
data0 = vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
len_to_deq = clib_min (ctx->left_to_snd, ctx->deq_per_first_buf);
/* Ensure snd_mss does accurately reflect the amount of data we can push
* in a segment. This also makes sure that options are updated according to
* the current state of the connection. */
/* Ensure snd_mss does accurately reflect the amount of data we can push
* in a segment. This also makes sure that options are updated according to
* the current state of the connection. */
- tcp_update_snd_mss (tc);
+ tcp_update_burst_snd_vars (tc);
output nodes */
vlib_frame_t *ip_lookup_tx_frames[2]; /**< tx frames for ip 4/6
lookup nodes */
output nodes */
vlib_frame_t *ip_lookup_tx_frames[2]; /**< tx frames for ip 4/6
lookup nodes */
+ CLIB_CACHE_LINE_ALIGN_MARK (cacheline1);
+ u8 cached_opts[40]; /**< cached 'on the wire'
+ options for bursts */
} tcp_worker_ctx_t;
typedef struct _tcp_main
} tcp_worker_ctx_t;
typedef struct _tcp_main
void tcp_send_syn (tcp_connection_t * tc);
void tcp_send_fin (tcp_connection_t * tc);
void tcp_init_mss (tcp_connection_t * tc);
void tcp_send_syn (tcp_connection_t * tc);
void tcp_send_fin (tcp_connection_t * tc);
void tcp_init_mss (tcp_connection_t * tc);
-void tcp_update_snd_mss (tcp_connection_t * tc);
+void tcp_update_burst_snd_vars (tcp_connection_t * tc);
void tcp_update_rto (tcp_connection_t * tc);
void tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4);
void tcp_flush_frames_to_output (u8 thread_index);
void tcp_update_rto (tcp_connection_t * tc);
void tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4);
void tcp_flush_frames_to_output (u8 thread_index);
- * Update snd_mss to reflect the effective segment size that we can send
- * by taking into account all TCP options, including SACKs
+ * Update burst send vars
+ *
+ * - Updates snd_mss to reflect the effective segment size that we can send
+ * by taking into account all TCP options, including SACKs.
+ * - Cache 'on the wire' options for reuse
+ * - Updates receive window which can be reused for a burst.
+ *
+ * This should *only* be called when doing bursts
-tcp_update_snd_mss (tcp_connection_t * tc)
+tcp_update_burst_snd_vars (tcp_connection_t * tc)
+ tcp_main_t *tm = &tcp_main;
+
/* Compute options to be used for connection. These may be reused when
* sending data or to compute the effective mss (snd_mss) */
/* Compute options to be used for connection. These may be reused when
* sending data or to compute the effective mss (snd_mss) */
- tc->snd_opts_len =
- tcp_make_options (tc, &tc->snd_opts, TCP_STATE_ESTABLISHED);
+ tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
+ TCP_STATE_ESTABLISHED);
/* XXX check if MTU has been updated */
tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
ASSERT (tc->snd_mss > 0);
/* XXX check if MTU has been updated */
tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
ASSERT (tc->snd_mss > 0);
+
+ tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
+ &tc->snd_opts);
+
+ tcp_update_rcv_wnd (tc);
*/
always_inline void
tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
*/
always_inline void
tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
- tcp_state_t next_state, u8 compute_opts)
+ tcp_state_t next_state, u8 compute_opts, u8 maybe_burst)
{
u32 advertise_wnd, data_len;
{
u32 advertise_wnd, data_len;
- u8 tcp_hdr_opts_len, opts_write_len, flags;
+ u8 tcp_hdr_opts_len, flags;
+ tcp_main_t *tm = &tcp_main;
- data_len = b->current_length + b->total_length_not_including_first_buffer;
- ASSERT (!b->total_length_not_including_first_buffer
- || (b->flags & VLIB_BUFFER_NEXT_PRESENT));
+ data_len = b->current_length;
+ if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
+ data_len += b->total_length_not_including_first_buffer;
+
vnet_buffer (b)->tcp.flags = 0;
vnet_buffer (b)->tcp.flags = 0;
+ vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
if (compute_opts)
tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
if (compute_opts)
tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
- advertise_wnd = tcp_window_to_advertise (tc, next_state);
+
+ if (maybe_burst)
+ advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
+ else
+ advertise_wnd = tcp_window_to_advertise (tc, next_state);
+
flags = tcp_make_state_flags (tc, next_state);
flags = tcp_make_state_flags (tc, next_state);
- /* Push header and options */
th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
tc->rcv_nxt, tcp_hdr_opts_len, flags,
advertise_wnd);
th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
tc->rcv_nxt, tcp_hdr_opts_len, flags,
advertise_wnd);
- opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
- ASSERT (opts_write_len == tc->snd_opts_len);
- vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
+ if (maybe_burst)
+ {
+ clib_memcpy ((u8 *) (th + 1),
+ tm->wrk_ctx[tc->c_thread_index].cached_opts,
+ tc->snd_opts_len);
+ }
+ else
+ {
+ u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
+ ASSERT (len == tc->snd_opts_len);
+ }
/*
* Update connection variables
/*
* Update connection variables
u32
tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b)
{
u32
tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b)
{
- tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0);
+ tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, /* compute opts */ 0,
+ /* burst */ 1);
tc->snd_una_max = tc->snd_nxt;
ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
tc->snd_una_max = tc->snd_nxt;
ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
max_deq_bytes);
ASSERT (n_bytes == max_deq_bytes);
b[0]->current_length = n_bytes;
max_deq_bytes);
ASSERT (n_bytes == max_deq_bytes);
b[0]->current_length = n_bytes;
- tcp_push_hdr_i (tc, *b, tc->state, 0);
+ tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
}
/* Split mss into multiple buffers */
else
}
/* Split mss into multiple buffers */
else
b[0]->total_length_not_including_first_buffer += n_peeked;
}
b[0]->total_length_not_including_first_buffer += n_peeked;
}
- tcp_push_hdr_i (tc, *b, tc->state, 0);
+ tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
|| tc->snd_nxt == tc->snd_una_max
|| tc->rto_boff > 1));
|| tc->snd_nxt == tc->snd_una_max
|| tc->rto_boff > 1));
- tcp_push_hdr_i (tc, b, tc->state, 0);
+ tcp_push_hdr_i (tc, b, tc->state, /* compute opts */ 0, /* burst */ 0);
tc->snd_una_max = tc->snd_nxt;
tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
tc->snd_una_max = tc->snd_nxt;
tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);