tcp: add per worker ctx structure

Change-Id: I28d3c31bdc4255a4ca223d80bcf44709fb39f4ed
Signed-off-by: Florin Coras <fcoras@cisco.com>

diff --git a/src/vnet/tcp/tcp.c b/src/vnet/tcp/tcp.c
index 854577b..3017361 100644 (file)
--- a/src/vnet/tcp/tcp.c
+++ b/src/vnet/tcp/tcp.c
@@ -1063,7 +1063,8 @@ void
 tcp_update_time (f64 now, u8 thread_index)
 {
   tcp_set_time_now (thread_index);
-  tw_timer_expire_timers_16t_2w_512sl (&tcp_main.timer_wheels[thread_index],
+  tw_timer_expire_timers_16t_2w_512sl (&tcp_main.
+                                      wrk_ctx[thread_index].timer_wheel,
                                       now);
   tcp_flush_frames_to_output (thread_index);
 }
@@ -1203,7 +1204,7 @@ tcp_initialize_timer_wheels (tcp_main_t * tm)
   tw_timer_wheel_16t_2w_512sl_t *tw;
   /* *INDENT-OFF* */
   foreach_vlib_main (({
-    tw = &tm->timer_wheels[ii];
+    tw = &tm->wrk_ctx[ii].timer_wheel;
     tw_timer_wheel_init_16t_2w_512sl (tw, tcp_expired_timers_dispatch,
                                      100e-3 /* timer period 100ms */ , ~0);
     tw->last_run_time = vlib_time_now (this_vlib_main);
@@ -1272,13 +1273,6 @@ tcp_main_enable (vlib_main_t * vm)
     pool_init_fixed (tm->half_open_connections,
                     tm->preallocated_half_open_connections);
 
-  /* Initialize per worker thread tx buffers (used for control messages) */
-  vec_validate (tm->tx_buffers, num_threads - 1);
-
-  /* Initialize timer wheels */
-  vec_validate (tm->timer_wheels, num_threads - 1);
-  tcp_initialize_timer_wheels (tm);
-
   /* Initialize clocks per tick for TCP timestamp. Used to compute
    * monotonically increasing timestamps. */
   tm->tstamp_ticks_per_clock = vm->clib_time.seconds_per_clock
@@ -1289,15 +1283,12 @@ tcp_main_enable (vlib_main_t * vm)
       clib_spinlock_init (&tm->half_open_lock);
     }
 
-  vec_validate (tm->tx_frames[0], num_threads - 1);
-  vec_validate (tm->tx_frames[1], num_threads - 1);
-  vec_validate (tm->ip_lookup_tx_frames[0], num_threads - 1);
-  vec_validate (tm->ip_lookup_tx_frames[1], num_threads - 1);
+  vec_validate (tm->wrk_ctx, num_threads - 1);
+  tcp_initialize_timer_wheels (tm);
 
   tm->bytes_per_buffer = vlib_buffer_free_list_buffer_size
     (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
 
-  vec_validate (tm->time_now, num_threads - 1);
   return error;
 }
 

diff --git a/src/vnet/tcp/tcp.h b/src/vnet/tcp/tcp.h
index 5673c8c..ba57e3a 100644 (file)
--- a/src/vnet/tcp/tcp.h
+++ b/src/vnet/tcp/tcp.h
@@ -358,6 +358,18 @@ typedef struct _tcp_lookup_dispatch
   u8 next, error;
 } tcp_lookup_dispatch_t;
 
+typedef struct tcp_worker_ctx_
+{
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
+  u32 time_now;                                        /**< worker time */
+  tw_timer_wheel_16t_2w_512sl_t timer_wheel;   /**< worker timer wheel */
+  u32 *tx_buffers;                             /**< tx buffer free list */
+  vlib_frame_t *tx_frames[2];                  /**< tx frames for tcp 4/6
+                                                    output nodes */
+  vlib_frame_t *ip_lookup_tx_frames[2];                /**< tx frames for ip 4/6
+                                                    lookup nodes */
+} tcp_worker_ctx_t;
+
 typedef struct _tcp_main
 {
   /* Per-worker thread tcp connection pools */
@@ -371,17 +383,9 @@ typedef struct _tcp_main
 
   u8 log2_tstamp_clocks_per_tick;
   f64 tstamp_ticks_per_clock;
-  u32 *time_now;
-
-  /** per-worker tx buffer free lists */
-  u32 **tx_buffers;
-  /** per-worker tx frames to tcp 4/6 output nodes */
-  vlib_frame_t **tx_frames[2];
-  /** per-worker tx frames to ip 4/6 lookup nodes */
-  vlib_frame_t **ip_lookup_tx_frames[2];
 
-  /* Per worker-thread timer wheel for connections timers */
-  tw_timer_wheel_16t_2w_512sl_t *timer_wheels;
+  /** per-worker context */
+  tcp_worker_ctx_t *wrk_ctx;
 
   /* Pool of half-open connections on which we've sent a SYN */
   tcp_connection_t *half_open_connections;
@@ -657,15 +661,15 @@ u32 tcp_sack_list_bytes (tcp_connection_t * tc);
 always_inline u32
 tcp_time_now (void)
 {
-  return tcp_main.time_now[vlib_get_thread_index ()];
+  return tcp_main.wrk_ctx[vlib_get_thread_index ()].time_now;
 }
 
 always_inline u32
 tcp_set_time_now (u32 thread_index)
 {
-  tcp_main.time_now[thread_index] = clib_cpu_time_now ()
+  tcp_main.wrk_ctx[thread_index].time_now = clib_cpu_time_now ()
     * tcp_main.tstamp_ticks_per_clock;
-  return tcp_main.time_now[thread_index];
+  return tcp_main.wrk_ctx[thread_index].time_now;
 }
 
 u32 tcp_session_push_header (transport_connection_t * tconn,
@@ -693,9 +697,10 @@ tcp_timer_set (tcp_connection_t * tc, u8 timer_id, u32 interval)
 {
   ASSERT (tc->c_thread_index == vlib_get_thread_index ());
   ASSERT (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID);
-  tc->timers[timer_id]
-    = tw_timer_start_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index],
-                                  tc->c_c_index, timer_id, interval);
+  tc->timers[timer_id] =
+    tw_timer_start_16t_2w_512sl (&tcp_main.
+                                wrk_ctx[tc->c_thread_index].timer_wheel,
+                                tc->c_c_index, timer_id, interval);
 }
 
 always_inline void
@@ -705,7 +710,8 @@ tcp_timer_reset (tcp_connection_t * tc, u8 timer_id)
   if (tc->timers[timer_id] == TCP_TIMER_HANDLE_INVALID)
     return;
 
-  tw_timer_stop_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index],
+  tw_timer_stop_16t_2w_512sl (&tcp_main.
+                             wrk_ctx[tc->c_thread_index].timer_wheel,
                              tc->timers[timer_id]);
   tc->timers[timer_id] = TCP_TIMER_HANDLE_INVALID;
 }
@@ -715,10 +721,12 @@ tcp_timer_update (tcp_connection_t * tc, u8 timer_id, u32 interval)
 {
   ASSERT (tc->c_thread_index == vlib_get_thread_index ());
   if (tc->timers[timer_id] != TCP_TIMER_HANDLE_INVALID)
-    tw_timer_stop_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index],
+    tw_timer_stop_16t_2w_512sl (&tcp_main.
+                               wrk_ctx[tc->c_thread_index].timer_wheel,
                                tc->timers[timer_id]);
   tc->timers[timer_id] =
-    tw_timer_start_16t_2w_512sl (&tcp_main.timer_wheels[tc->c_thread_index],
+    tw_timer_start_16t_2w_512sl (&tcp_main.
+                                wrk_ctx[tc->c_thread_index].timer_wheel,
                                 tc->c_c_index, timer_id, interval);
 }
 

diff --git a/src/vnet/tcp/tcp_output.c b/src/vnet/tcp/tcp_output.c
index 641277b..e29eb6d 100644 (file)
--- a/src/vnet/tcp/tcp_output.c
+++ b/src/vnet/tcp/tcp_output.c
@@ -448,16 +448,16 @@ always_inline int
 tcp_alloc_tx_buffers (tcp_main_t * tm, u8 thread_index, u16 * n_bufs,
                      u32 wanted)
 {
+  tcp_worker_ctx_t *ctx = &tm->wrk_ctx[thread_index];
   vlib_main_t *vm = vlib_get_main ();
   u32 n_alloc;
 
   ASSERT (wanted > *n_bufs);
-  vec_validate_aligned (tm->tx_buffers[thread_index], wanted - 1,
-                       CLIB_CACHE_LINE_BYTES);
-  n_alloc = vlib_buffer_alloc (vm, &tm->tx_buffers[thread_index][*n_bufs],
+  vec_validate_aligned (ctx->tx_buffers, wanted - 1, CLIB_CACHE_LINE_BYTES);
+  n_alloc = vlib_buffer_alloc (vm, &ctx->tx_buffers[*n_bufs],
                               wanted - *n_bufs);
   *n_bufs += n_alloc;
-  _vec_len (tm->tx_buffers[thread_index]) = *n_bufs;
+  _vec_len (ctx->tx_buffers) = *n_bufs;
   return n_alloc;
 }
 
@@ -465,7 +465,8 @@ always_inline int
 tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx)
 {
   u32 thread_index = vlib_get_thread_index ();
-  u16 n_bufs = vec_len (tm->tx_buffers[thread_index]);
+  tcp_worker_ctx_t *ctx = &tm->wrk_ctx[thread_index];
+  u16 n_bufs = vec_len (ctx->tx_buffers);
 
   TCP_DBG_BUFFER_ALLOC_MAYBE_FAIL (thread_index);
 
@@ -477,8 +478,8 @@ tcp_get_free_buffer_index (tcp_main_t * tm, u32 * bidx)
          return -1;
        }
     }
-  *bidx = tm->tx_buffers[thread_index][--n_bufs];
-  _vec_len (tm->tx_buffers[thread_index]) = n_bufs;
+  *bidx = ctx->tx_buffers[--n_bufs];
+  _vec_len (ctx->tx_buffers) = n_bufs;
   return 0;
 }
 
@@ -647,12 +648,12 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
   next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
   tcp_trajectory_add_start (b, 1);
 
-  f = tm->ip_lookup_tx_frames[!is_ip4][thread_index];
+  f = tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4];
   if (!f)
     {
       f = vlib_get_frame_to_node (vm, next_index);
       ASSERT (f);
-      tm->ip_lookup_tx_frames[!is_ip4][thread_index] = f;
+      tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = f;
     }
 
   to_next = vlib_frame_vector_args (f);
@@ -661,7 +662,7 @@ tcp_enqueue_to_ip_lookup_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
   if (flush || f->n_vectors == VLIB_FRAME_SIZE)
     {
       vlib_put_frame_to_node (vm, next_index, f);
-      tm->ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
+      tm->wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = 0;
     }
 }
 
@@ -698,12 +699,12 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
   tcp_trajectory_add_start (b, 2);
 
   /* Get frame to v4/6 output node */
-  f = tm->tx_frames[!is_ip4][thread_index];
+  f = tm->wrk_ctx[thread_index].tx_frames[!is_ip4];
   if (!f)
     {
       f = vlib_get_frame_to_node (vm, next_index);
       ASSERT (f);
-      tm->tx_frames[!is_ip4][thread_index] = f;
+      tm->wrk_ctx[thread_index].tx_frames[!is_ip4] = f;
     }
   to_next = vlib_frame_vector_args (f);
   to_next[f->n_vectors] = bi;
@@ -711,7 +712,7 @@ tcp_enqueue_to_output_i (vlib_main_t * vm, vlib_buffer_t * b, u32 bi,
   if (flush || f->n_vectors == VLIB_FRAME_SIZE)
     {
       vlib_put_frame_to_node (vm, next_index, f);
-      tm->tx_frames[!is_ip4][thread_index] = 0;
+      tm->wrk_ctx[thread_index].tx_frames[!is_ip4] = 0;
     }
 }
 
@@ -1007,13 +1008,14 @@ tcp_send_syn (tcp_connection_t * tc)
 void
 tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 {
-  if (tcp_main.tx_frames[!is_ip4][thread_index])
+  if (tcp_main.wrk_ctx[thread_index].tx_frames[!is_ip4])
     {
       u32 next_index;
       next_index = is_ip4 ? tcp4_output_node.index : tcp6_output_node.index;
       vlib_put_frame_to_node (vm, next_index,
-                             tcp_main.tx_frames[!is_ip4][thread_index]);
-      tcp_main.tx_frames[!is_ip4][thread_index] = 0;
+                             tcp_main.
+                             wrk_ctx[thread_index].tx_frames[!is_ip4]);
+      tcp_main.wrk_ctx[thread_index].tx_frames[!is_ip4] = 0;
     }
 }
 
@@ -1023,14 +1025,15 @@ tcp_flush_frame_to_output (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 always_inline void
 tcp_flush_frame_to_ip_lookup (vlib_main_t * vm, u8 thread_index, u8 is_ip4)
 {
-  if (tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index])
+  if (tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4])
     {
       u32 next_index;
       next_index = is_ip4 ? ip4_lookup_node.index : ip6_lookup_node.index;
       vlib_put_frame_to_node (vm, next_index,
-                             tcp_main.ip_lookup_tx_frames[!is_ip4]
-                             [thread_index]);
-      tcp_main.ip_lookup_tx_frames[!is_ip4][thread_index] = 0;
+                             tcp_main.
+                             wrk_ctx[thread_index].ip_lookup_tx_frames
+                             [!is_ip4]);
+      tcp_main.wrk_ctx[thread_index].ip_lookup_tx_frames[!is_ip4] = 0;
     }
 }
 
@@ -1264,7 +1267,7 @@ tcp_prepare_retransmit_segment (tcp_connection_t * tc, u32 offset,
 
       /* Make sure we have enough buffers */
       n_bufs_per_seg = ceil ((double) seg_size / tm->bytes_per_buffer);
-      available_bufs = vec_len (tm->tx_buffers[thread_index]);
+      available_bufs = vec_len (tm->wrk_ctx[thread_index].tx_buffers);
       if (n_bufs_per_seg > available_bufs)
        {
          tcp_alloc_tx_buffers (tm, thread_index, &available_bufs,