vlib: interrupt mode support for pre-input nodes

Type: improvement
Change-Id: Ic6e60597d2be63e3a0ae4399a81dbbd72392f30d
Signed-off-by: Damjan Marion <damarion@cisco.com>

diff --git a/src/vlib/main.c b/src/vlib/main.c
index 0f67bb8..9d34aa1 100644 (file)
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1471,8 +1471,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   else
     cpu_time_now = clib_cpu_time_now ();
 
-  /* Pre-allocate interupt runtime indices and lock. */
-  vec_validate_aligned (nm->pending_interrupts, 0, CLIB_CACHE_LINE_BYTES);
+  nm->pending_interrupts = 0;
 
   /* Pre-allocate expired nodes. */
   if (!nm->polling_threshold_vector_length)
@@ -1505,6 +1504,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   while (1)
     {
       vlib_node_runtime_t *n;
+      u8 pending_interrupts;
 
       if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0))
        {
@@ -1552,6 +1552,27 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
                                      /* frame */ 0,
                                      cpu_time_now);
 
+      pending_interrupts =
+       __atomic_load_n (&nm->pending_interrupts, __ATOMIC_ACQUIRE);
+
+      if (pending_interrupts)
+       {
+         int int_num = -1;
+         nm->pending_interrupts = 0;
+
+         while ((int_num = clib_interrupt_get_next (
+                   nm->pre_input_node_interrupts, int_num)) != -1)
+           {
+             vlib_node_runtime_t *n;
+             clib_interrupt_clear (nm->pre_input_node_interrupts, int_num);
+             n = vec_elt_at_index (
+               nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num);
+             cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT,
+                                           VLIB_NODE_STATE_INTERRUPT,
+                                           /* frame */ 0, cpu_time_now);
+           }
+       }
+
       /* Next process input nodes. */
       vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])
        cpu_time_now = dispatch_node (vm, n,
@@ -1563,16 +1584,15 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
       if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0))
        vm->queue_signal_callback (vm);
 
-      if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE))
+      if (pending_interrupts)
        {
          int int_num = -1;
-         *nm->pending_interrupts = 0;
 
-         while ((int_num =
-                   clib_interrupt_get_next (nm->interrupts, int_num)) != -1)
+         while ((int_num = clib_interrupt_get_next (nm->input_node_interrupts,
+                                                    int_num)) != -1)
            {
              vlib_node_runtime_t *n;
-             clib_interrupt_clear (nm->interrupts, int_num);
+             clib_interrupt_clear (nm->input_node_interrupts, int_num);
              n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
                                    int_num);
              cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT,

diff --git a/src/vlib/node.c b/src/vlib/node.c
index 4d1ef98..8b726ff 100644 (file)
--- a/src/vlib/node.c
+++ b/src/vlib/node.c
@@ -530,7 +530,10 @@ vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt,
        vec_add2_aligned (nm->nodes_by_type[n->type], rt, 1,
                          /* align */ CLIB_CACHE_LINE_BYTES);
        if (n->type == VLIB_NODE_TYPE_INPUT)
-         clib_interrupt_resize (&nm->interrupts,
+         clib_interrupt_resize (&nm->input_node_interrupts,
+                                vec_len (nm->nodes_by_type[n->type]));
+       else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+         clib_interrupt_resize (&nm->pre_input_node_interrupts,
                                 vec_len (nm->nodes_by_type[n->type]));
        n->runtime_index = rt - nm->nodes_by_type[n->type];
       }

diff --git a/src/vlib/node.h b/src/vlib/node.h
index e40208b..955ffb9 100644 (file)
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -690,8 +690,9 @@ typedef struct
   vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE];
 
   /* Node runtime indices for input nodes with pending interrupts. */
-  void *interrupts;
-  volatile u32 *pending_interrupts;
+  void *input_node_interrupts;
+  void *pre_input_node_interrupts;
+  volatile u8 pending_interrupts;
 
   /* Input nodes are switched from/to interrupt to/from polling mode
      when average vector length goes above/below polling/interrupt

diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index a1fca14..66f079c 100644 (file)
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -252,15 +252,21 @@ vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index)
 {
   vlib_node_main_t *nm = &vm->node_main;
   vlib_node_t *n = vec_elt (nm->nodes, node_index);
+  void *interrupts;
 
-  ASSERT (n->type == VLIB_NODE_TYPE_INPUT);
+  if (n->type == VLIB_NODE_TYPE_INPUT)
+    interrupts = nm->input_node_interrupts;
+  else if (n->type == VLIB_NODE_TYPE_PRE_INPUT)
+    interrupts = nm->pre_input_node_interrupts;
+  else
+    ASSERT (0);
 
   if (vm != vlib_get_main ())
-    clib_interrupt_set_atomic (nm->interrupts, n->runtime_index);
+    clib_interrupt_set_atomic (interrupts, n->runtime_index);
   else
-    clib_interrupt_set (nm->interrupts, n->runtime_index);
+    clib_interrupt_set (interrupts, n->runtime_index);
 
-  __atomic_store_n (nm->pending_interrupts, 1, __ATOMIC_RELEASE);
+  __atomic_store_n (&nm->pending_interrupts, 1, __ATOMIC_RELEASE);
 }
 
 always_inline vlib_process_t *

diff --git a/src/vlib/threads.c b/src/vlib/threads.c
index adf225b..4b70642 100644 (file)
--- a/src/vlib/threads.c
+++ b/src/vlib/threads.c
@@ -699,8 +699,11 @@ start_workers (vlib_main_t * vm)
                vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
                                 CLIB_CACHE_LINE_BYTES);
              clib_interrupt_init (
-               &nm_clone->interrupts,
+               &nm_clone->input_node_interrupts,
                vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+             clib_interrupt_init (
+               &nm_clone->pre_input_node_interrupts,
+               vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
              vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
              {
                vlib_node_t *n = vlib_get_node (vm, rt->node_index);
@@ -1022,8 +1025,11 @@ vlib_worker_thread_node_refork (void)
     vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT],
                     CLIB_CACHE_LINE_BYTES);
   clib_interrupt_resize (
-    &nm_clone->interrupts,
+    &nm_clone->input_node_interrupts,
     vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]));
+  clib_interrupt_resize (
+    &nm_clone->pre_input_node_interrupts,
+    vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT]));
 
   vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])
   {

diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c
index 9c7c54f..39c124c 100644 (file)
--- a/src/vlib/unix/input.c
+++ b/src/vlib/unix/input.c
@@ -250,7 +250,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                while (nanosleep (&ts, &tsrem) < 0)
                  ts = tsrem;
                if (*vlib_worker_threads->wait_at_barrier ||
-                   *nm->pending_interrupts)
+                   nm->pending_interrupts)
                  goto done;
              }
          }