dpdk: improve buffer alloc perfomance

[vpp.git] / src / vlib / threads.c

diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index be8daa6..bbe94c7 100644 (file)
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -783,7 +783,6 @@ start_workers (vlib_main_t * vm)
       for (i = 0; i < vec_len (tm->registrations); i++)
        {
          vlib_node_main_t *nm, *nm_clone;
-         vlib_buffer_main_t *bm_clone;
          vlib_buffer_free_list_t *fl_clone, *fl_orig;
          vlib_buffer_free_list_t *orig_freelist_pool;
          int k;
@@ -822,21 +821,25 @@ start_workers (vlib_main_t * vm)
              /* Fork vlib_global_main et al. Look for bugs here */
              oldheap = clib_mem_set_heap (w->thread_mheap);
 
-             vm_clone = clib_mem_alloc (sizeof (*vm_clone));
+             vm_clone = clib_mem_alloc_aligned (sizeof (*vm_clone),
+                                                CLIB_CACHE_LINE_BYTES);
              clib_memcpy (vm_clone, vlib_mains[0], sizeof (*vm_clone));
 
              vm_clone->thread_index = worker_thread_index;
              vm_clone->heap_base = w->thread_mheap;
-             vm_clone->mbuf_alloc_list = 0;
              vm_clone->init_functions_called =
                hash_create (0, /* value bytes */ 0);
+             vm_clone->pending_rpc_requests = 0;
+             vec_validate (vm_clone->pending_rpc_requests, 0);
+             _vec_len (vm_clone->pending_rpc_requests) = 0;
              memset (&vm_clone->random_buffer, 0,
                      sizeof (vm_clone->random_buffer));
 
              nm = &vlib_mains[0]->node_main;
              nm_clone = &vm_clone->node_main;
              /* fork next frames array, preserving node runtime indices */
-             nm_clone->next_frames = vec_dup (nm->next_frames);
+             nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
+                                                      CLIB_CACHE_LINE_BYTES);
              for (j = 0; j < vec_len (nm_clone->next_frames); j++)
                {
                  vlib_next_frame_t *nf = &nm_clone->next_frames[j];
@@ -873,7 +876,8 @@ start_workers (vlib_main_t * vm)
                  n++;
                }
              nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
-               vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+               vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
+                                CLIB_CACHE_LINE_BYTES);
              vec_foreach (rt,
                           nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
              {
@@ -887,7 +891,8 @@ start_workers (vlib_main_t * vm)
              }
 
              nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
-               vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
+               vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
+                                CLIB_CACHE_LINE_BYTES);
              vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
              {
                vlib_node_t *n = vlib_get_node (vm, rt->node_index);
@@ -899,7 +904,8 @@ start_workers (vlib_main_t * vm)
                                         n->runtime_data_bytes));
              }
 
-             nm_clone->processes = vec_dup (nm->processes);
+             nm_clone->processes = vec_dup_aligned (nm->processes,
+                                                    CLIB_CACHE_LINE_BYTES);
 
              /* zap the (per worker) frame freelists, etc */
              nm_clone->frame_sizes = 0;
@@ -910,25 +916,23 @@ start_workers (vlib_main_t * vm)
              clib_mem_set_heap (oldheap);
              vec_add1_aligned (vlib_mains, vm_clone, CLIB_CACHE_LINE_BYTES);
 
-             vm_clone->error_main.counters =
-               vec_dup (vlib_mains[0]->error_main.counters);
-             vm_clone->error_main.counters_last_clear =
-               vec_dup (vlib_mains[0]->error_main.counters_last_clear);
+             vm_clone->error_main.counters = vec_dup_aligned
+               (vlib_mains[0]->error_main.counters, CLIB_CACHE_LINE_BYTES);
+             vm_clone->error_main.counters_last_clear = vec_dup_aligned
+               (vlib_mains[0]->error_main.counters_last_clear,
+                CLIB_CACHE_LINE_BYTES);
 
              /* Fork the vlib_buffer_main_t free lists, etc. */
-             bm_clone = vec_dup (vm_clone->buffer_main);
-             vm_clone->buffer_main = bm_clone;
-
-             orig_freelist_pool = bm_clone->buffer_free_list_pool;
-             bm_clone->buffer_free_list_pool = 0;
+             orig_freelist_pool = vm_clone->buffer_free_list_pool;
+             vm_clone->buffer_free_list_pool = 0;
 
             /* *INDENT-OFF* */
             pool_foreach (fl_orig, orig_freelist_pool,
                           ({
-                            pool_get_aligned (bm_clone->buffer_free_list_pool,
+                            pool_get_aligned (vm_clone->buffer_free_list_pool,
                                               fl_clone, CLIB_CACHE_LINE_BYTES);
                             ASSERT (fl_orig - orig_freelist_pool
-                                    == fl_clone - bm_clone->buffer_free_list_pool);
+                                    == fl_clone - vm_clone->buffer_free_list_pool);
 
                             fl_clone[0] = fl_orig[0];
                             fl_clone->buffers = 0;
@@ -1094,7 +1098,8 @@ vlib_worker_thread_node_refork (void)
 
   nm_clone = &vm_clone->node_main;
   vec_free (nm_clone->next_frames);
-  nm_clone->next_frames = vec_dup (nm->next_frames);
+  nm_clone->next_frames = vec_dup_aligned (nm->next_frames,
+                                          CLIB_CACHE_LINE_BYTES);
 
   for (j = 0; j < vec_len (nm_clone->next_frames); j++)
     {
@@ -1162,7 +1167,8 @@ vlib_worker_thread_node_refork (void)
   /* re-clone internal nodes */
   old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL];
   nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL] =
-    vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL]);
+    vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INTERNAL],
+                    CLIB_CACHE_LINE_BYTES);
 
   vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INTERNAL])
   {
@@ -1188,7 +1194,8 @@ vlib_worker_thread_node_refork (void)
   /* re-clone input nodes */
   old_rt = nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT];
   nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT] =
-    vec_dup (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]);
+    vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
+                    CLIB_CACHE_LINE_BYTES);
 
   vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
   {
@@ -1211,7 +1218,8 @@ vlib_worker_thread_node_refork (void)
 
   vec_free (old_rt);
 
-  nm_clone->processes = vec_dup (nm->processes);
+  nm_clone->processes = vec_dup_aligned (nm->processes,
+                                        CLIB_CACHE_LINE_BYTES);
 }
 
 void