init / exit function ordering

[vpp.git] / src / vlib / unix / input.c

diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c
index 0c29844..43bb206 100644 (file)
--- a/src/vlib/unix/input.c
+++ b/src/vlib/unix/input.c
@@ -52,6 +52,7 @@
 
 typedef struct
 {
+  CLIB_CACHE_LINE_ALIGN_MARK (cacheline0);
   int epoll_fd;
   struct epoll_event *epoll_events;
   int n_epoll_fds;
@@ -144,12 +145,34 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     vlib_node_main_t *nm = &vm->node_main;
     u32 ticks_until_expiration;
     f64 timeout;
+    f64 now;
     int timeout_ms = 0, max_timeout_ms = 10;
     f64 vector_rate = vlib_last_vectors_per_main_loop (vm);
 
+    if (is_main == 0)
+      now = vlib_time_now (vm);
+
+    /*
+     * If we've been asked for a fixed-sleep between main loop polls,
+     * do so right away.
+     */
+    if (PREDICT_FALSE (is_main && um->poll_sleep_usec))
+      {
+       struct timespec ts, tsrem;
+       timeout = 0;
+       timeout_ms = 0;
+       node->input_main_loops_per_call = 0;
+       ts.tv_sec = 0;
+       ts.tv_nsec = 1000 * um->poll_sleep_usec;
+
+       while (nanosleep (&ts, &tsrem) < 0)
+         {
+           ts = tsrem;
+         }
+      }
     /* If we're not working very hard, decide how long to sleep */
-    if (is_main && vector_rate < 2 && vm->api_queue_nonempty == 0
-       && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
+    else if (is_main && vector_rate < 2 && vm->api_queue_nonempty == 0
+            && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
       {
        ticks_until_expiration = TW (tw_timer_first_expires_in_ticks)
          ((TWT (tw_timer_wheel) *) nm->timing_wheel);
@@ -175,8 +198,9 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
          }
        node->input_main_loops_per_call = 0;
       }
-    else if (is_main == 0 && vector_rate < 2 &&
-            nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
+    else if (is_main == 0 && vector_rate < 2
+            && (vlib_global_main.time_last_barrier_release + 0.5 < now)
+            && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
       {
        timeout = 10e-3;
        timeout_ms = max_timeout_ms;
@@ -204,12 +228,32 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
                                      em->epoll_events,
                                      vec_len (em->epoll_events), timeout_ms);
          }
+
       }
     else
       {
+       /*
+        * Worker thread, no epoll fd's, sleep for 100us at a time
+        * and check for a barrier sync request
+        */
        if (timeout_ms)
-         usleep (timeout_ms * 1000);
-       return 0;
+         {
+           struct timespec ts, tsrem;
+           f64 limit = now + (f64) timeout_ms * 1e-3;
+
+           while (vlib_time_now (vm) < limit)
+             {
+               /* Sleep for 100us at a time */
+               ts.tv_sec = 0;
+               ts.tv_nsec = 1000 * 100;
+
+               while (nanosleep (&ts, &tsrem) < 0)
+                 ts = tsrem;
+               if (*vlib_worker_threads->wait_at_barrier)
+                 goto done;
+             }
+         }
+       goto done;
       }
   }
 
@@ -219,7 +263,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        vlib_panic_with_error (vm, clib_error_return_unix (0, "epoll_wait"));
 
       /* non fatal error (e.g. EINTR). */
-      return 0;
+      goto done;
     }
 
   em->epoll_waits += 1;
@@ -232,18 +276,51 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
       clib_error_t *errors[4];
       int n_errors = 0;
 
-      if (PREDICT_TRUE (!(e->events & EPOLLERR)))
+      if (PREDICT_FALSE (pool_is_free (fm->file_pool, f)))
+       {
+         /*
+          * Under rare scenerop, epoll may still post us events for the
+          * deleted file descriptor. We just deal with it and throw away the
+          * events for the corresponding file descriptor.
+          */
+         if (e->events & EPOLLIN)
+           {
+             errors[n_errors] =
+               clib_error_return (0, "epoll event EPOLLIN dropped due "
+                                  "to free index %u", i);
+             n_errors++;
+           }
+         if (e->events & EPOLLOUT)
+           {
+             errors[n_errors] =
+               clib_error_return (0, "epoll event EPOLLOUT dropped due "
+                                  "to free index %u", i);
+             n_errors++;
+           }
+         if (e->events & EPOLLERR)
+           {
+             errors[n_errors] =
+               clib_error_return (0, "epoll event EPOLLERR dropped due "
+                                  "to free index %u", i);
+             n_errors++;
+           }
+       }
+      else if (PREDICT_TRUE (!(e->events & EPOLLERR)))
        {
          if (e->events & EPOLLIN)
            {
-             errors[n_errors] = f->read_function (f);
              f->read_events++;
+             errors[n_errors] = f->read_function (f);
+             /* Make sure f is valid if the file pool moves */
+             if (pool_is_free_index (fm->file_pool, i))
+               continue;
+             f = pool_elt_at_index (fm->file_pool, i);
              n_errors += errors[n_errors] != 0;
            }
          if (e->events & EPOLLOUT)
            {
-             errors[n_errors] = f->write_function (f);
              f->write_events++;
+             errors[n_errors] = f->write_function (f);
              n_errors += errors[n_errors] != 0;
            }
        }
@@ -251,8 +328,8 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        {
          if (f->error_function)
            {
-             errors[n_errors] = f->error_function (f);
              f->error_events++;
+             errors[n_errors] = f->error_function (f);
              n_errors += errors[n_errors] != 0;
            }
          else
@@ -266,6 +343,13 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
        }
     }
 
+done:
+  if (PREDICT_FALSE (vm->cpu_id != clib_get_current_cpu_id ()))
+    {
+      vm->cpu_id = clib_get_current_cpu_id ();
+      vm->numa_node = clib_get_current_numa_node ();
+    }
+
   return 0;
 }
 
@@ -327,10 +411,15 @@ VLIB_INIT_FUNCTION (linux_epoll_input_init);
 static clib_error_t *
 unix_input_init (vlib_main_t * vm)
 {
-  return vlib_call_init_function (vm, linux_epoll_input_init);
+  return 0;
 }
 
-VLIB_INIT_FUNCTION (unix_input_init);
+/* *INDENT-OFF* */
+VLIB_INIT_FUNCTION (unix_input_init) =
+{
+  .runs_before = VLIB_INITS ("linux_epoll_input_init"),
+};
+/* *INDENT-ON* */
 
 /*
  * fd.io coding-style-patch-verification: ON