Perf tune handoff queue polling

Change-Id: I5cfa0f6eee67156bf87907fcf8a39f16d68a0905
Signed-off-by: Dave Barach <dave@barachs.net>

diff --git a/src/vlib/buffer_node.h b/src/vlib/buffer_node.h
index 7e48878..5ded506 100644 (file)
--- a/src/vlib/buffer_node.h
+++ b/src/vlib/buffer_node.h
@@ -507,7 +507,6 @@ vlib_buffer_enqueue_to_thread (vlib_main_t * vm, u32 frame_queue_index,
 
       if (next_thread_index != current_thread_index)
        {
-
          if (drop_on_congestion &&
              is_vlib_frame_queue_congested
              (frame_queue_index, next_thread_index, fqm->queue_hi_thresh,
@@ -518,6 +517,7 @@ vlib_buffer_enqueue_to_thread (vlib_main_t * vm, u32 frame_queue_index,
              n_drop++;
              goto next;
            }
+         vlib_mains[next_thread_index]->check_frame_queues = 1;
 
          if (hf)
            hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;

diff --git a/src/vlib/main.c b/src/vlib/main.c
index 4dcf63e..c80aa5a 100644 (file)
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1679,6 +1679,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   u64 cpu_time_now;
   vlib_frame_queue_main_t *fqm;
   u32 *last_node_runtime_indices = 0;
+  u32 frame_queue_check_counter = 0;
 
   /* Initialize pending node vector. */
   if (is_main)
@@ -1734,8 +1735,26 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       if (!is_main)
        {
          vlib_worker_thread_barrier_check ();
-         vec_foreach (fqm, tm->frame_queue_mains)
-           vlib_frame_queue_dequeue (vm, fqm);
+         if (PREDICT_FALSE (vm->check_frame_queues +
+                            frame_queue_check_counter))
+           {
+             u32 processed = 0;
+
+             if (vm->check_frame_queues)
+               {
+                 frame_queue_check_counter = 100;
+                 vm->check_frame_queues = 0;
+               }
+
+             vec_foreach (fqm, tm->frame_queue_mains)
+               processed += vlib_frame_queue_dequeue (vm, fqm);
+
+             /* No handoff queue work found? */
+             if (processed)
+               frame_queue_check_counter = 100;
+             else
+               frame_queue_check_counter--;
+           }
          if (PREDICT_FALSE (vm->worker_thread_main_loop_callback != 0))
            ((void (*)(vlib_main_t *)) vm->worker_thread_main_loop_callback)
              (vm);

diff --git a/src/vlib/main.h b/src/vlib/main.h
index 2c5bb45..d2c4213 100644 (file)
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -236,6 +236,9 @@ typedef struct vlib_main_t
   /* Earliest barrier can be closed again */
   f64 barrier_no_close_before;
 
+  /* Need to check the frame queues */
+  volatile uword check_frame_queues;
+
   /* RPC requests, main thread only */
   uword *pending_rpc_requests;
   uword *processing_rpc_requests;