X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvnet%2Ftcp%2Ftcp_output.c;h=7d7c32ad210ad0f163cfd051106d81cd1e271a4f;hb=c977e7c100f5727183f9452e2c0d086623a21d15;hp=e29eb6d7f9aea724e1c08d6113f383880837bcb0;hpb=2c4144367cb4fd9f44853e2bb974e78eccff3cc7;p=vpp.git

diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index e29eb6d7f9a..7d7c32ad210 100644
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -118,12 +118,13 @@ tcp_initial_wnd_unscaled (tcp_connection_t * tc)
 u32
 tcp_initial_window_to_advertise (tcp_connection_t * tc)
 {
+  tcp_main_t *tm = &tcp_main;
   u32 max_fifo;
 
   /* Initial wnd for SYN. Fifos are not allocated yet.
    * Use some predefined value. For SYN-ACK we still want the
    * scale to be computed in the same way */
-  max_fifo = TCP_MAX_RX_FIFO_SIZE;
+  max_fifo = tm->max_rx_fifo ? tm->max_rx_fifo : TCP_MAX_RX_FIFO_SIZE;
 
   tc->rcv_wscale = tcp_window_compute_scale (max_fifo);
   tc->rcv_wnd = tcp_initial_wnd_unscaled (tc);
@@ -131,30 +132,7 @@ tcp_initial_window_to_advertise (tcp_connection_t * tc)
   return clib_min (tc->rcv_wnd, TCP_WND_MAX);
 }
 
-/**
- * Compute and return window to advertise, scaled as per RFC1323
- */
-u32
-tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
-{
-  if (state < TCP_STATE_ESTABLISHED)
-    return tcp_initial_window_to_advertise (tc);
-
-  tcp_update_rcv_wnd (tc);
-
-  if (tc->rcv_wnd == 0)
-    {
-      tc->flags |= TCP_CONN_SENT_RCV_WND0;
-    }
-  else
-    {
-      tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
-    }
-
-  return tc->rcv_wnd >> tc->rcv_wscale;
-}
-
-void
+static void
 tcp_update_rcv_wnd (tcp_connection_t * tc)
 {
   i32 observed_wnd;
@@ -200,10 +178,33 @@ tcp_update_rcv_wnd (tcp_connection_t * tc)
   tc->rcv_wnd = clib_min (wnd, TCP_WND_MAX << tc->rcv_wscale);
 }
 
+/**
+ * Compute and return window to advertise, scaled as per RFC1323
+ */
+static u32
+tcp_window_to_advertise (tcp_connection_t * tc, tcp_state_t state)
+{
+  if (state < TCP_STATE_ESTABLISHED)
+    return tcp_initial_window_to_advertise (tc);
+
+  tcp_update_rcv_wnd (tc);
+
+  if (tc->rcv_wnd == 0)
+    {
+      tc->flags |= TCP_CONN_SENT_RCV_WND0;
+    }
+  else
+    {
+      tc->flags &= ~TCP_CONN_SENT_RCV_WND0;
+    }
+
+  return tc->rcv_wnd >> tc->rcv_wscale;
+}
+
 /**
  * Write TCP options to segment.
  */
-u32
+static u32
 tcp_options_write (u8 * data, tcp_options_t * opts)
 {
   u32 opts_len = 0;
@@ -286,7 +287,7 @@ tcp_options_write (u8 * data, tcp_options_t * opts)
   return opts_len;
 }
 
-always_inline int
+static int
 tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
 {
   u8 len = 0;
@@ -315,7 +316,7 @@ tcp_make_syn_options (tcp_options_t * opts, u8 wnd_scale)
   return len;
 }
 
-always_inline int
+static int
 tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
 {
   u8 len = 0;
@@ -350,7 +351,7 @@ tcp_make_synack_options (tcp_connection_t * tc, tcp_options_t * opts)
   return len;
 }
 
-always_inline int
+static int
 tcp_make_established_options (tcp_connection_t * tc, tcp_options_t * opts)
 {
   u8 len = 0;
@@ -403,20 +404,33 @@ tcp_make_options (tcp_connection_t * tc, tcp_options_t * opts,
 }
 
 /**
- * Update snd_mss to reflect the effective segment size that we can send
- * by taking into account all TCP options, including SACKs
+ * Update burst send vars
+ *
+ * - Updates snd_mss to reflect the effective segment size that we can send
+ * by taking into account all TCP options, including SACKs.
+ * - Cache 'on the wire' options for reuse
+ * - Updates receive window which can be reused for a burst.
+ *
+ * This should *only* be called when doing bursts
  */
 void
-tcp_update_snd_mss (tcp_connection_t * tc)
+tcp_update_burst_snd_vars (tcp_connection_t * tc)
 {
+  tcp_main_t *tm = &tcp_main;
+
   /* Compute options to be used for connection. These may be reused when
    * sending data or to compute the effective mss (snd_mss) */
-  tc->snd_opts_len =
-    tcp_make_options (tc, &tc->snd_opts, TCP_STATE_ESTABLISHED);
+  tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts,
+				       TCP_STATE_ESTABLISHED);
 
   /* XXX check if MTU has been updated */
   tc->snd_mss = clib_min (tc->mss, tc->rcv_opts.mss) - tc->snd_opts_len;
   ASSERT (tc->snd_mss > 0);
+
+  tcp_options_write (tm->wrk_ctx[tc->c_thread_index].cached_opts,
+		     &tc->snd_opts);
+
+  tcp_update_rcv_wnd (tc);
 }
 
 void
@@ -444,7 +458,7 @@ tcp_init_mss (tcp_connection_t * tc)
     tc->snd_mss -= TCP_OPTION_LEN_TIMESTAMP;
 }
 
-always_inline int
+static int
 tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs,
 		      u32 wanted)
 {
@@ -483,7 +497,7 @@ tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx)
   return 0;
 }
 
-always_inline void *
+static void *
 tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 {
   if (b->flags & VLIB_BUFFER_NEXT_PRESENT)
@@ -499,7 +513,7 @@ tcp_reuse_buffer (vlib_main_t * vm, vlib_buffer_t * b)
   return vlib_buffer_make_headroom (b, MAX_HDRS_LEN);
 }
 
-always_inline void *
+static void *
 tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 {
   ASSERT ((b->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
@@ -516,7 +530,7 @@ tcp_init_buffer (vlib_main_t * vm, vlib_buffer_t * b)
 /**
  * Prepare ACK
  */
-void
+static void
 tcp_make_ack_i (tcp_connection_t * tc, vlib_buffer_t * b, tcp_state_t state,
 		u8 flags)
 {
@@ -666,14 +680,14 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
     }
 }
 
-always_inline void
+static void
 tcp_enqueue_to_ip_lookup_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
 			      u8 is_ip4, u32 fib_index)
 {
   tcp_enqueue_to_ip_lookup_i (vm, b, bi, is_ip4, fib_index, 1);
 }
 
-always_inline void
+static void
 tcp_enqueue_to_ip_lookup (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
 			  u8 is_ip4, u32 fib_index)
 {
@@ -716,20 +730,20 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
     }
 }
 
-always_inline void
+static void
 tcp_enqueue_to_output (vlib_main_t * vm, vlib_buffer_t * b, u32 bi, u8 is_ip4)
 {
   tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 0);
 }
 
-always_inline void
+static void
 tcp_enqueue_to_output_now (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
 			   u8 is_ip4)
 {
   tcp_enqueue_to_output_i (vm, b, bi, is_ip4, 1);
 }
 
-int
+static int
 tcp_make_reset_in_place (vlib_main_t * vm, vlib_buffer_t * b0,
 			 tcp_state_t state, u8 thread_index, u8 is_ip4)
 {
@@ -938,7 +952,7 @@ tcp_send_reset (tcp_connection_t * tc)
   TCP_EVT_DBG (TCP_EVT_RST_SENT, tc);
 }
 
-void
+static void
 tcp_push_ip_hdr (tcp_main_t * tm, tcp_connection_t * tc, vlib_buffer_t * b)
 {
   tcp_header_t *th = vlib_buffer_get_current (b);
@@ -1022,7 +1036,7 @@ tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 /**
  * Flush ip lookup tx frames populated by timer pops
  */
-always_inline void
+static void
 tcp_flush_frame_to_ip_lookup (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 {
   if (tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4])
@@ -1062,16 +1076,25 @@ tcp_send_fin (tcp_connection_t * tc)
   u32 bi;
   u8 fin_snt = 0;
 
-  tcp_retransmit_timer_force_update (tc);
-  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
-    return;
-  b = vlib_get_buffer (vm, bi);
-  tcp_init_buffer (vm, b);
   fin_snt = tc->flags & TCP_CONN_FINSNT;
   if (fin_snt)
     tc->snd_nxt = tc->snd_una;
+
+  if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
+    {
+      /* Out of buffers so program fin retransmit ASAP */
+      tcp_timer_update (tc, TCP_TIMER_RETRANSMIT, 1);
+      goto post_enqueue;
+    }
+
+  tcp_retransmit_timer_force_update (tc);
+  b = vlib_get_buffer (vm, bi);
+  tcp_init_buffer (vm, b);
   tcp_make_fin (tc, b);
   tcp_enqueue_to_output_now (vm, b, bi, tc->c_is_ip4);
+  TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
+
+post_enqueue:
   if (!fin_snt)
     {
       tc->flags |= TCP_CONN_FINSNT;
@@ -1084,7 +1107,6 @@ tcp_send_fin (tcp_connection_t * tc)
     {
       tc->snd_nxt = tc->snd_una_max;
     }
-  TCP_EVT_DBG (TCP_EVT_FIN_SENT, tc);
 }
 
 always_inline u8
@@ -1114,34 +1136,49 @@ tcp_make_state_flags (tcp_connection_t * tc, tcp_state_t next_state)
 /**
  * Push TCP header and update connection variables
  */
-static void
+always_inline void
 tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
-		tcp_state_t next_state, u8 compute_opts)
+		tcp_state_t next_state, u8 compute_opts, u8 maybe_burst)
 {
   u32 advertise_wnd, data_len;
-  u8 tcp_hdr_opts_len, opts_write_len, flags;
+  u8 tcp_hdr_opts_len, flags;
+  tcp_main_t *tm = &tcp_main;
   tcp_header_t *th;
 
-  data_len = b->current_length + b->total_length_not_including_first_buffer;
-  ASSERT (!b->total_length_not_including_first_buffer
-	  || (b->flags & VLIB_BUFFER_NEXT_PRESENT));
+  data_len = b->current_length;
+  if (PREDICT_FALSE (b->flags & VLIB_BUFFER_NEXT_PRESENT))
+    data_len += b->total_length_not_including_first_buffer;
+
   vnet_buffer (b)->tcp.flags = 0;
+  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
 
   if (compute_opts)
     tc->snd_opts_len = tcp_make_options (tc, &tc->snd_opts, tc->state);
 
   tcp_hdr_opts_len = tc->snd_opts_len + sizeof (tcp_header_t);
-  advertise_wnd = tcp_window_to_advertise (tc, next_state);
+
+  if (maybe_burst)
+    advertise_wnd = tc->rcv_wnd >> tc->rcv_wscale;
+  else
+    advertise_wnd = tcp_window_to_advertise (tc, next_state);
+
   flags = tcp_make_state_flags (tc, next_state);
 
-  /* Push header and options */
   th = vlib_buffer_push_tcp (b, tc->c_lcl_port, tc->c_rmt_port, tc->snd_nxt,
 			     tc->rcv_nxt, tcp_hdr_opts_len, flags,
 			     advertise_wnd);
-  opts_write_len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
 
-  ASSERT (opts_write_len == tc->snd_opts_len);
-  vnet_buffer (b)->tcp.connection_index = tc->c_c_index;
+  if (maybe_burst)
+    {
+      clib_memcpy ((u8 *) (th + 1),
+		   tm->wrk_ctx[tc->c_thread_index].cached_opts,
+		   tc->snd_opts_len);
+    }
+  else
+    {
+      u8 len = tcp_options_write ((u8 *) (th + 1), &tc->snd_opts);
+      ASSERT (len == tc->snd_opts_len);
+    }
 
   /*
    * Update connection variables
@@ -1153,6 +1190,30 @@ tcp_push_hdr_i (tcp_connection_t * tc, vlib_buffer_t * b,
   TCP_EVT_DBG (TCP_EVT_PKTIZE, tc);
 }
 
+u32
+tcp_push_header (tcp_connection_t * tc, vlib_buffer_t * b)
+{
+  tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, /* compute opts */ 0,
+		  /* burst */ 1);
+  tc->snd_una_max = tc->snd_nxt;
+  ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd
+		   + tcp_fastrecovery_sent_1_smss (tc) * tc->snd_mss));
+  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
+  /* If not tracking an ACK, start tracking */
+  if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
+    {
+      tc->rtt_ts = tcp_time_now ();
+      tc->rtt_seq = tc->snd_nxt;
+    }
+  if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
+    {
+      tcp_retransmit_timer_set (tc);
+      tc->rto_boff = 0;
+    }
+  tcp_trajectory_add_start (b, 3);
+  return 0;
+}
+
 void
 tcp_send_ack (tcp_connection_t * tc)
 {
@@ -1195,7 +1256,7 @@ tcp_timer_delack_handler (u32 index)
  * @return the number of bytes in the segment or 0 if there's nothing to
  *         retransmit
  */
-u32
+static u32
 tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset,
 				u32 max_deq_bytes, vlib_buffer_t ** b)
 {
@@ -1254,7 +1315,9 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset,
 					   max_deq_bytes);
       ASSERT (n_bytes == max_deq_bytes);
       b[0]->current_length = n_bytes;
-      tcp_push_hdr_i (tc, *b, tc->state, 0);
+      tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
+      if (seq_gt (tc->snd_nxt, tc->snd_una_max))
+	tc->snd_una_max = tc->snd_nxt;
     }
   /* Split mss into multiple buffers */
   else
@@ -1317,7 +1380,9 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset,
 	  b[0]->total_length_not_including_first_buffer += n_peeked;
 	}
 
-      tcp_push_hdr_i (tc, *b, tc->state, 0);
+      tcp_push_hdr_i (tc, *b, tc->state, /* compute opts */ 0, /* burst */ 0);
+      if (seq_gt (tc->snd_nxt, tc->snd_una_max))
+	tc->snd_una_max = tc->snd_nxt;
     }
 
   ASSERT (n_bytes > 0);
@@ -1356,7 +1421,7 @@ tcp_rxt_timeout_cc (tcp_connection_t * tc)
   tcp_recovery_on (tc);
 }
 
-static void
+static inline void
 tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
 {
   tcp_main_t *tm = vnet_get_tcp_main ();
@@ -1396,15 +1461,19 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
 	  return;
 	}
 
-      /* Shouldn't be here */
+      /* Shouldn't be here. This condition is tricky because it has to take
+       * into account boff > 0 due to persist timeout. */
       if ((tc->rto_boff == 0 && tc->snd_una == tc->snd_una_max)
-	  || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)))
+	  || (tc->rto_boff > 0 && seq_geq (tc->snd_una, tc->snd_congestion)
+	      && !tcp_flight_size (tc)))
 	{
-	  tcp_recovery_off (tc);
+	  ASSERT (!tcp_in_recovery (tc));
+	  tc->rto_boff = 0;
 	  return;
 	}
 
-      /* We're not in recovery so make sure rto_boff is 0 */
+      /* We're not in recovery so make sure rto_boff is 0. Can be non 0 due
+       * to persist timer timeout */
       if (!tcp_in_recovery (tc) && tc->rto_boff > 0)
 	{
 	  tc->rto_boff = 0;
@@ -1419,10 +1488,15 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
       if (tc->rto_boff == 1)
 	tcp_rxt_timeout_cc (tc);
 
+      /* If we've sent beyond snd_congestion, update it */
+      if (seq_gt (tc->snd_una_max, tc->snd_congestion))
+	tc->snd_congestion = tc->snd_una_max;
+
       tc->snd_una_max = tc->snd_nxt = tc->snd_una;
       tc->rto = clib_min (tc->rto << 1, TCP_RTO_MAX);
 
-      /* Send one segment. Note that n_bytes may be zero due to buffer shortfall  */
+      /* Send one segment. Note that n_bytes may be zero due to buffer
+       * shortfall */
       n_bytes = tcp_prepare_retransmit_segment (tc, 0, tc->snd_mss, &b);
 
       /* TODO be less aggressive about this */
@@ -1430,7 +1504,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
 
       if (n_bytes == 0)
 	{
-	  tcp_retransmit_timer_set (tc);
+	  tcp_retransmit_timer_force_update (tc);
 	  return;
 	}
 
@@ -1441,7 +1515,7 @@ tcp_timer_retransmit_handler_i (u32 index, u8 is_syn)
 	tc->snd_rxt_ts = tcp_time_now ();
 
       tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
-      tcp_retransmit_timer_update (tc);
+      tcp_retransmit_timer_force_update (tc);
     }
   /* Retransmit for SYN */
   else if (tc->state == TCP_STATE_SYN_SENT)
@@ -1577,7 +1651,10 @@ tcp_timer_persist_handler (u32 index)
    * Try to force the first unsent segment (or buffer)
    */
   if (PREDICT_FALSE (tcp_get_free_buffer_index (tm, &bi)))
-    return;
+    {
+      tcp_persist_timer_set (tc);
+      return;
+    }
   b = vlib_get_buffer (vm, bi);
   data = tcp_init_buffer (vm, b);
 
@@ -1591,7 +1668,7 @@ tcp_timer_persist_handler (u32 index)
 			   || tc->snd_nxt == tc->snd_una_max
 			   || tc->rto_boff > 1));
 
-  tcp_push_hdr_i (tc, b, tc->state, 0);
+  tcp_push_hdr_i (tc, b, tc->state, /* compute opts */ 0, /* burst */ 0);
   tc->snd_una_max = tc->snd_nxt;
   tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
   tcp_enqueue_to_output (vm, b, bi, tc->c_is_ip4);
@@ -1755,7 +1832,7 @@ tcp_fast_retransmit (tcp_connection_t * tc)
     tcp_fast_retransmit_no_sack (tc);
 }
 
-always_inline u32
+static u32
 tcp_session_has_ooo_data (tcp_connection_t * tc)
 {
   stream_session_t *s = session_get (tc->c_s_index, tc->c_thread_index);
@@ -1817,7 +1894,7 @@ tcp46_output_trace_frame (vlib_main_t * vm, vlib_node_runtime_t * node,
     }
 }
 
-static inline void
+always_inline void
 tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
 		    tcp_connection_t * tc0, u8 is_ip4)
 {
@@ -1845,7 +1922,7 @@ tcp_output_push_ip (vlib_main_t * vm, vlib_buffer_t * b0,
     }
 }
 
-static inline void
+always_inline void
 tcp_output_handle_packet (tcp_connection_t * tc0, vlib_buffer_t * b0,
 			  u32 * error0, u16 * next0, u8 is_ip4)
 {
@@ -1945,8 +2022,8 @@ tcp46_output_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
 
       if (n_left_from > 1)
 	{
-	  vlib_prefetch_buffer_header (b[0], STORE);
-	  CLIB_PREFETCH (b[0]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+	  vlib_prefetch_buffer_header (b[1], STORE);
+	  CLIB_PREFETCH (b[1]->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
 	}
 
       next[0] = TCP_OUTPUT_NEXT_IP_LOOKUP;
@@ -2022,29 +2099,6 @@ VLIB_REGISTER_NODE (tcp6_output_node) =
 
 VLIB_NODE_FUNCTION_MULTIARCH (tcp6_output_node, tcp6_output);
 
-u32
-tcp_session_push_header (transport_connection_t * tconn, vlib_buffer_t * b)
-{
-  tcp_connection_t *tc = (tcp_connection_t *) tconn;
-  tcp_push_hdr_i (tc, b, TCP_STATE_ESTABLISHED, 0);
-  tc->snd_una_max = tc->snd_nxt;
-  ASSERT (seq_leq (tc->snd_una_max, tc->snd_una + tc->snd_wnd));
-  tcp_validate_txf_size (tc, tc->snd_una_max - tc->snd_una);
-  /* If not tracking an ACK, start tracking */
-  if (tc->rtt_ts == 0 && !tcp_in_cong_recovery (tc))
-    {
-      tc->rtt_ts = tcp_time_now ();
-      tc->rtt_seq = tc->snd_nxt;
-    }
-  if (PREDICT_FALSE (!tcp_timer_is_active (tc, TCP_TIMER_RETRANSMIT)))
-    {
-      tcp_retransmit_timer_set (tc);
-      tc->rto_boff = 0;
-    }
-  tcp_trajectory_add_start (b, 3);
-  return 0;
-}
-
 typedef enum _tcp_reset_next
 {
   TCP_RESET_NEXT_DROP,