vlib: improve automatic core pinning

[vpp.git] / src / vlib / threads.c

diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index 713e192..3994afc 100644 (file)
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -179,7 +179,9 @@ vlib_thread_init (vlib_main_t * vm)
   u32 n_vlib_mains = 1;
   u32 first_index = 1;
   u32 i;
-  uword *avail_cpu;
+  pid_t pid;
+  uword *avail_cpu, *affinity_cpu;
+  uword n_cpus;
   u32 stats_num_worker_threads_dir_index;
 
   stats_num_worker_threads_dir_index =
@@ -190,16 +192,39 @@ vlib_thread_init (vlib_main_t * vm)
   tm->cpu_core_bitmap = os_get_online_cpu_core_bitmap ();
   tm->cpu_socket_bitmap = os_get_online_cpu_node_bitmap ();
 
+  /* get bitmap of active cpu cores vpp has affinity to */
+  pid = getpid ();
+  tm->cpu_affinity_bitmap = os_get_cpu_affinity_bitmap (pid);
+
+  /* if fetching affinity fails, return online cpu core bmp */
+  if (tm->cpu_affinity_bitmap == 0)
+    tm->cpu_affinity_bitmap = os_get_online_cpu_core_bitmap ();
+
   avail_cpu = clib_bitmap_dup (tm->cpu_core_bitmap);
+  affinity_cpu = clib_bitmap_dup (tm->cpu_affinity_bitmap);
 
   /* skip cores */
+  n_cpus = clib_bitmap_count_set_bits (avail_cpu);
+  if (tm->skip_cores >= n_cpus)
+    return clib_error_return (0, "skip-core greater than available cpus");
+  n_cpus = clib_bitmap_count_set_bits (affinity_cpu);
+  if (tm->skip_cores >= n_cpus)
+    return clib_error_return (0, "skip-core greater than affinity cpus");
+
   for (i = 0; i < tm->skip_cores; i++)
     {
-      uword c = clib_bitmap_first_set (avail_cpu);
+      uword c;
+      c = clib_bitmap_first_set (avail_cpu);
       if (c == ~0)
        return clib_error_return (0, "no available cpus to skip");
 
       avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
+
+      c = clib_bitmap_first_set (affinity_cpu);
+      if (c == ~0)
+       return clib_error_return (0, "no available env cpus to skip");
+
+      affinity_cpu = clib_bitmap_set (affinity_cpu, c, 0);
     }
 
   /* grab cpu for main thread */
@@ -209,6 +234,17 @@ vlib_thread_init (vlib_main_t * vm)
        return clib_error_return (0, "cpu %u is not available to be used"
                                  " for the main thread", tm->main_lcore);
       avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
+      affinity_cpu = clib_bitmap_set (affinity_cpu, tm->main_lcore, 0);
+    }
+  /* if auto enabled, grab first cpu vpp has affinity to for main thread */
+  else if (tm->use_main_core_auto)
+    {
+      uword c = clib_bitmap_first_set (affinity_cpu);
+      if (c != ~0)
+       tm->main_lcore = c;
+
+      avail_cpu = clib_bitmap_set (avail_cpu, tm->main_lcore, 0);
+      affinity_cpu = clib_bitmap_set (affinity_cpu, tm->main_lcore, 0);
     }
 
   /* assume that there is socket 0 only if there is no data from sysfs */
@@ -289,13 +325,23 @@ vlib_thread_init (vlib_main_t * vm)
        }
       else
        {
+         /* for automatic pinning, use cpu affinity list */
+         uword n_env_cpu = 0;
+         n_env_cpu = clib_bitmap_count_set_bits (affinity_cpu);
+
+         if (n_env_cpu < tr->count)
+           return clib_error_return (0,
+                                     "no available cpus to be used for"
+                                     " the '%s' thread #%u",
+                                     tr->name, n_env_cpu);
+
          for (j = 0; j < tr->count; j++)
            {
              /* Do not use CPU 0 by default - leave it to the host and IRQs */
-             uword avail_c0 = clib_bitmap_get (avail_cpu, 0);
-             avail_cpu = clib_bitmap_set (avail_cpu, 0, 0);
+             uword avail_c0 = clib_bitmap_get (affinity_cpu, 0);
+             affinity_cpu = clib_bitmap_set (affinity_cpu, 0, 0);
 
-             uword c = clib_bitmap_first_set (avail_cpu);
+             uword c = clib_bitmap_first_set (affinity_cpu);
              /* Use CPU 0 as a last resort */
              if (c == ~0 && avail_c0)
                {
@@ -309,14 +355,15 @@ vlib_thread_init (vlib_main_t * vm)
                                          " the '%s' thread #%u",
                                          tr->name, tr->count);
 
-             avail_cpu = clib_bitmap_set (avail_cpu, 0, avail_c0);
-             avail_cpu = clib_bitmap_set (avail_cpu, c, 0);
+             affinity_cpu = clib_bitmap_set (affinity_cpu, 0, avail_c0);
+             affinity_cpu = clib_bitmap_set (affinity_cpu, c, 0);
              tr->coremask = clib_bitmap_set (tr->coremask, c, 1);
            }
        }
     }
 
   clib_bitmap_free (avail_cpu);
+  clib_bitmap_free (affinity_cpu);
 
   tm->n_vlib_mains = n_vlib_mains;
   vlib_stats_set_gauge (stats_num_worker_threads_dir_index, n_vlib_mains - 1);
@@ -1118,6 +1165,7 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input)
   tm->sched_policy = ~0;
   tm->sched_priority = ~0;
   tm->main_lcore = ~0;
+  tm->use_main_core_auto = 0;
 
   tr = tm->next;
 
@@ -1133,6 +1181,8 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input)
        tm->use_pthreads = 1;
       else if (unformat (input, "thread-prefix %v", &tm->thread_prefix))
        ;
+      else if (unformat (input, "main-core auto"))
+       tm->use_main_core_auto = 1;
       else if (unformat (input, "main-core %u", &tm->main_lcore))
        ;
       else if (unformat (input, "skip-cores %u", &tm->skip_cores))
@@ -1191,6 +1241,13 @@ cpu_config (vlib_main_t * vm, unformat_input_t * input)
        break;
     }
 
+  if (tm->main_lcore != ~0 && tm->use_main_core_auto)
+    {
+      return clib_error_return (
+       0, "cannot set both 'main-core %u' and 'main-core auto'",
+       tm->main_lcore);
+    }
+
   if (tm->sched_priority != ~0)
     {
       if (tm->sched_policy == SCHED_FIFO || tm->sched_policy == SCHED_RR)