tcp/session: sprinkle prefetches

Change-Id: Idef3c665580c13d72e99f43d16b8b13cc6ab746f
Signed-off-by: Florin Coras <fcoras@cisco.com>

diff --git a/src/vnet/session-apps/echo_server.c b/src/vnet/session-apps/echo_server.c
index 9f34d4c..0bde2e4 100644 (file)
--- a/src/vnet/session-apps/echo_server.c
+++ b/src/vnet/session-apps/echo_server.c
@@ -375,6 +375,7 @@ echo_server_create (vlib_main_t * vm, u8 * appns_id, u64 appns_flags,
   vec_validate (esm->rx_buf, num_threads - 1);
   vec_validate (esm->rx_retries, num_threads - 1);
 
+  esm->rcv_buffer_size = clib_max (esm->rcv_buffer_size, esm->fifo_size);
   for (i = 0; i < num_threads; i++)
     vec_validate (esm->rx_buf[i], esm->rcv_buffer_size);
 

diff --git a/src/vnet/session/session_node.c b/src/vnet/session/session_node.c
index 9cd0ef1..b8f429e 100644 (file)
--- a/src/vnet/session/session_node.c
+++ b/src/vnet/session/session_node.c
@@ -144,7 +144,7 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
   u8 *data0;
   int i, n_bytes_read;
   u32 n_bytes_per_buf, deq_per_buf, deq_per_first_buf;
-  u32 buffers_allocated, buffers_allocated_this_call;
+  u32 bufs_alloc, bufs_now;
 
   next_index = next0 = smm->session_type_to_next[s0->session_type];
 
@@ -218,23 +218,19 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
        {
          vec_validate (smm->tx_buffers[thread_index],
                        n_bufs + n_bufs_per_frame - 1);
-         buffers_allocated = 0;
+         bufs_alloc = 0;
          do
            {
-             buffers_allocated_this_call = vlib_buffer_alloc (vm,
-                                                              &smm->tx_buffers
-                                                              [thread_index]
-                                                              [n_bufs +
-                                                               buffers_allocated],
-                                                              n_bufs_per_frame
-                                                              -
-                                                              buffers_allocated);
-             buffers_allocated += buffers_allocated_this_call;
+             bufs_now =
+               vlib_buffer_alloc (vm,
+                                  &smm->tx_buffers[thread_index][n_bufs +
+                                                                 bufs_alloc],
+                                  n_bufs_per_frame - bufs_alloc);
+             bufs_alloc += bufs_now;
            }
-         while (buffers_allocated_this_call > 0
-                && ((buffers_allocated + n_bufs < n_bufs_per_frame)));
+         while (bufs_now > 0 && ((bufs_alloc + n_bufs < n_bufs_per_frame)));
 
-         n_bufs += buffers_allocated;
+         n_bufs += bufs_alloc;
          _vec_len (smm->tx_buffers[thread_index]) = n_bufs;
 
          if (PREDICT_FALSE (n_bufs < n_bufs_per_frame))
@@ -252,6 +248,15 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
           * Handle first buffer in chain separately
           */
 
+         len_to_deq0 = clib_min (left_to_snd0, deq_per_first_buf);
+         if (left_to_snd0 > len_to_deq0 && n_left_to_next > 1)
+           {
+             vlib_buffer_t *pb;
+             u32 pbi = smm->tx_buffers[thread_index][n_bufs - 2];
+             pb = vlib_get_buffer (vm, pbi);
+             vlib_prefetch_buffer_header (pb, LOAD);
+           }
+
          /* Get free buffer */
          ASSERT (n_bufs >= 1);
          bi0 = smm->tx_buffers[thread_index][--n_bufs];
@@ -264,11 +269,10 @@ session_tx_fifo_read_and_snd_i (vlib_main_t * vm, vlib_node_runtime_t * node,
 
          b0 = vlib_get_buffer (vm, bi0);
          b0->error = 0;
-         b0->flags |= VNET_BUFFER_F_LOCALLY_ORIGINATED;
+         b0->flags = VNET_BUFFER_F_LOCALLY_ORIGINATED;
          b0->current_data = 0;
          b0->total_length_not_including_first_buffer = 0;
 
-         len_to_deq0 = clib_min (left_to_snd0, deq_per_first_buf);
          data0 = vlib_buffer_make_headroom (b0, MAX_HDRS_LEN);
          if (peek_data)
            {

diff --git a/src/vnet/tcp/tcp_input.c b/src/vnet/tcp/tcp_input.c
index 86bce48..07f1838 100644 (file)
--- a/src/vnet/tcp/tcp_input.c
+++ b/src/vnet/tcp/tcp_input.c
@@ -1713,6 +1713,14 @@ tcp46_established_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          tcp_connection_t *tc0;
          u32 next0 = tcp_next_drop (is_ip4), error0 = TCP_ERROR_ENQUEUED;
 
+         if (n_left_from > 1)
+           {
+             vlib_buffer_t *pb;
+             pb = vlib_get_buffer (vm, from[1]);
+             vlib_prefetch_buffer_header (pb, LOAD);
+             CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, LOAD);
+           }
+
          bi0 = from[0];
          to_next[0] = bi0;
          from += 1;

diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index bbcbc91..2b5a5cd 100644 (file)
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -1771,6 +1771,14 @@ tcp46_output_inline (vlib_main_t * vm,
          tcp_header_t *th0 = 0;
          u32 error0 = TCP_ERROR_PKTS_SENT, next0 = TCP_OUTPUT_NEXT_IP_LOOKUP;
 
+         if (n_left_from > 1)
+           {
+             vlib_buffer_t *pb;
+             pb = vlib_get_buffer (vm, from[1]);
+             vlib_prefetch_buffer_header (pb, STORE);
+             CLIB_PREFETCH (pb->data, 2 * CLIB_CACHE_LINE_BYTES, STORE);
+           }
+
          bi0 = from[0];
          to_next[0] = bi0;
          from += 1;