vlib: always replace bootstrap vlib_mains

[vpp.git] / src / vlib / threads.c

diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 4df550e..ecca7e3 100644 (file)
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -626,6 +626,7 @@ vlib_get_thread_core_numa (vlib_worker_thread_t * w, unsigned cpu_id)
 static clib_error_t *
 vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
 {
+  clib_mem_main_t *mm = &clib_mem_main;
   vlib_thread_main_t *tm = &vlib_thread_main;
   void *(*fp_arg) (void *) = fp;
   void *numa_heap;
@@ -634,19 +635,22 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id)
   vlib_get_thread_core_numa (w, cpu_id);
 
   /* Set up NUMA-bound heap if indicated */
-  if (clib_per_numa_mheaps[w->numa_id] == 0)
+  if (mm->per_numa_mheaps[w->numa_id] == 0)
     {
       /* If the user requested a NUMA heap, create it... */
       if (tm->numa_heap_size)
        {
-         numa_heap = clib_mem_init_thread_safe_numa
-           (0 /* DIY */ , tm->numa_heap_size, w->numa_id);
-         clib_per_numa_mheaps[w->numa_id] = numa_heap;
+         clib_mem_set_numa_affinity (w->numa_id, 1 /* force */ );
+         numa_heap = clib_mem_create_heap (0 /* DIY */ , tm->numa_heap_size,
+                                           1 /* is_locked */ ,
+                                           "numa %u heap", w->numa_id);
+         clib_mem_set_default_numa_affinity ();
+         mm->per_numa_mheaps[w->numa_id] = numa_heap;
        }
       else
        {
          /* Or, use the main heap */
-         clib_per_numa_mheaps[w->numa_id] = w->thread_mheap;
+         mm->per_numa_mheaps[w->numa_id] = w->thread_mheap;
        }
     }
 
@@ -703,16 +707,15 @@ start_workers (vlib_main_t * vm)
   clib_callback_data_init (&vm->vlib_node_runtime_perf_callbacks,
                           &vm->worker_thread_main_loop_callback_lock);
 
-  if (n_vlib_mains > 1)
-    {
-      /* Replace hand-crafted length-1 vector with a real vector */
-      vlib_mains = 0;
+  /* Replace hand-crafted length-1 vector with a real vector */
+  vlib_mains = 0;
 
-      vec_validate_aligned (vlib_mains, tm->n_vlib_mains - 1,
-                           CLIB_CACHE_LINE_BYTES);
-      _vec_len (vlib_mains) = 0;
-      vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
+  vec_validate_aligned (vlib_mains, n_vlib_mains - 1, CLIB_CACHE_LINE_BYTES);
+  _vec_len (vlib_mains) = 0;
+  vec_add1_aligned (vlib_mains, vm, CLIB_CACHE_LINE_BYTES);
 
+  if (n_vlib_mains > 1)
+    {
       vlib_worker_threads->wait_at_barrier =
        clib_mem_alloc_aligned (sizeof (u32), CLIB_CACHE_LINE_BYTES);
       vlib_worker_threads->workers_at_barrier =
@@ -756,8 +759,10 @@ start_workers (vlib_main_t * vm)
              vec_add2 (vlib_worker_threads, w, 1);
              /* Currently unused, may not really work */
              if (tr->mheap_size)
-               w->thread_mheap = create_mspace (tr->mheap_size,
-                                                0 /* unlocked */ );
+               w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size,
+                                                       /* unlocked */ 0,
+                                                       "%s%d heap",
+                                                       tr->name, k);
              else
                w->thread_mheap = main_heap;
 
@@ -926,8 +931,10 @@ start_workers (vlib_main_t * vm)
              vec_add2 (vlib_worker_threads, w, 1);
              if (tr->mheap_size)
                {
-                 w->thread_mheap =
-                   create_mspace (tr->mheap_size, 0 /* locked */ );
+                 w->thread_mheap = clib_mem_create_heap (0, tr->mheap_size,
+                                                         /* locked */ 0,
+                                                         "%s%d heap",
+                                                         tr->name, j);
                }
              else
                w->thread_mheap = main_heap;
@@ -1444,6 +1451,18 @@ vlib_worker_thread_initial_barrier_sync_and_release (vlib_main_t * vm)
   *vlib_worker_threads->wait_at_barrier = 0;
 }
 
+/**
+ * Return true if the wroker thread barrier is held
+ */
+u8
+vlib_worker_thread_barrier_held (void)
+{
+  if (vec_len (vlib_mains) < 2)
+    return (1);
+
+  return (*vlib_worker_threads->wait_at_barrier == 1);
+}
+
 void
 vlib_worker_thread_barrier_sync_int (vlib_main_t * vm, const char *func_name)
 {
@@ -1647,6 +1666,41 @@ vlib_worker_thread_barrier_release (vlib_main_t * vm)
                         vm->clib_time.last_cpu_time, 1 /* leave */ );
 }
 
+/**
+ * Wait until each of the workers has been once around the track
+ */
+void
+vlib_worker_wait_one_loop (void)
+{
+  ASSERT (vlib_get_thread_index () == 0);
+
+  if (vec_len (vlib_mains) < 2)
+    return;
+
+  if (vlib_worker_thread_barrier_held ())
+    return;
+
+  u32 *counts = 0;
+  u32 ii;
+
+  vec_validate (counts, vec_len (vlib_mains) - 1);
+
+  /* record the current loop counts */
+  vec_foreach_index (ii, vlib_mains)
+    counts[ii] = vlib_mains[ii]->main_loop_count;
+
+  /* spin until each changes, apart from the main thread, or we'd be
+   * a while */
+  for (ii = 1; ii < vec_len (counts); ii++)
+    {
+      while (counts[ii] == vlib_mains[ii]->main_loop_count)
+       CLIB_PAUSE ();
+    }
+
+  vec_free (counts);
+  return;
+}
+
 /*
  * Check the frame queue to see if any frames are available.
  * If so, pull the packets off the frames and put them to