vlib: epoll on worker threads

This patch teaches worer threads to sleep and to be waken up by
kernel if there is activity on file desctiptors assigned to that thread.

It also adds counters to epoll file descriptors and new
debug cli 'show unix file'.

Change-Id: Iaf67869f4aa88ff5b0a08982e1c08474013107c4
Signed-off-by: Damjan Marion <damarion@cisco.com>

diff --git a/src/plugins/memif/memif.c b/src/plugins/memif/memif.c
index dc92e7e..7267ef2 100644 (file)
--- a/src/plugins/memif/memif.c
+++ b/src/plugins/memif/memif.c
@@ -221,6 +221,9 @@ memif_connect (memif_if_t * mif)
       {
        template.file_descriptor = mq->int_fd;
        template.private_data = (mif->dev_instance << 16) | (i & 0xFFFF);
+       template.description = format (0, "%U rx %u int",
+                                      format_memif_device_name,
+                                      mif->dev_instance, i);
        memif_file_add (&mq->int_clib_file_index, &template);
       }
     vnet_hw_interface_assign_rx_thread (vnm, mif->hw_if_index, i, ~0);
@@ -440,6 +443,9 @@ memif_process (vlib_main_t * vm, vlib_node_runtime_t * rt, vlib_frame_t * f)
                  t.file_descriptor = sock->fd;
                  t.private_data = mif->dev_instance;
                  memif_file_add (&sock->private_data, &t);
+                 t.description = format (0, "%U ctl",
+                                         format_memif_device_name,
+                                         mif->dev_instance);
                  hash_set (msf->dev_instance_by_fd, sock->fd, mif->dev_instance);
 
                  mif->flags |= MEMIF_IF_FLAG_CONNECTING;
@@ -805,6 +811,7 @@ memif_create_if (vlib_main_t * vm, memif_create_if_args_t * args)
       template.read_function = memif_conn_fd_accept_ready;
       template.file_descriptor = msf->sock->fd;
       template.private_data = mif->socket_file_index;
+      template.description = format (0, "memif listener %s", msf->filename);
       memif_file_add (&msf->sock->private_data, &template);
     }
 

diff --git a/src/plugins/memif/socket.c b/src/plugins/memif/socket.c
index 9aa4a2e..5e14f08 100644 (file)
--- a/src/plugins/memif/socket.c
+++ b/src/plugins/memif/socket.c
@@ -440,6 +440,9 @@ memif_msg_receive (memif_if_t ** mifp, clib_socket_t * sock, clib_file_t * uf)
       if ((err = memif_msg_receive_init (mifp, &msg, sock, uf->private_data)))
        return err;
       mif = *mifp;
+      vec_reset_length (uf->description);
+      uf->description = format (uf->description, "%U ctl",
+                               format_memif_device_name, mif->dev_instance);
       memif_msg_enq_ack (mif);
       break;
 
@@ -645,6 +648,7 @@ memif_conn_fd_accept_ready (clib_file_t * uf)
   template.error_function = memif_master_conn_fd_error;
   template.file_descriptor = client->fd;
   template.private_data = uf->private_data;
+  template.description = format (0, "memif in conn on %s", msf->filename);
 
   memif_file_add (&client->private_data, &template);
 

diff --git a/src/vlib/main.c b/src/vlib/main.c
index f915aa4..5efeac2 100644 (file)
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1493,13 +1493,12 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
        }
 
       /* Process pre-input nodes. */
-      if (is_main)
-       vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])
-         cpu_time_now = dispatch_node (vm, n,
-                                       VLIB_NODE_TYPE_PRE_INPUT,
-                                       VLIB_NODE_STATE_POLLING,
-                                       /* frame */ 0,
-                                       cpu_time_now);
+      vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])
+       cpu_time_now = dispatch_node (vm, n,
+                                     VLIB_NODE_TYPE_PRE_INPUT,
+                                     VLIB_NODE_STATE_POLLING,
+                                     /* frame */ 0,
+                                     cpu_time_now);
 
       /* Next process input nodes. */
       vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT])

diff --git a/src/vlib/unix/cli.c b/src/vlib/unix/cli.c
index 0cf4ed3..5838bbb 100644 (file)
--- a/src/vlib/unix/cli.c
+++ b/src/vlib/unix/cli.c
@@ -59,6 +59,7 @@
 #include <sys/ioctl.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <limits.h>
 
 /** ANSI escape code. */
 #define ESC "\x1b"
@@ -2556,6 +2557,9 @@ unix_cli_file_add (unix_cli_main_t * cm, char *name, int fd)
   clib_file_t template = { 0 };
   vlib_main_t *vm = um->vlib_main;
   vlib_node_t *n;
+  u8 *file_desc = 0;
+
+  file_desc = format (0, "%s", name);
 
   name = (char *) format (0, "unix-cli-%s", name);
 
@@ -2595,6 +2599,7 @@ unix_cli_file_add (unix_cli_main_t * cm, char *name, int fd)
   template.error_function = unix_cli_error_detected;
   template.file_descriptor = fd;
   template.private_data = cf - cm->cli_file_pool;
+  template.description = file_desc;
 
   cf->process_node_index = n->index;
   cf->clib_file_index = clib_file_add (fm, &template);
@@ -2867,6 +2872,7 @@ unix_cli_config (vlib_main_t * vm, unformat_input_t * input)
 
       template.read_function = unix_cli_listen_read_ready;
       template.file_descriptor = s->fd;
+      template.description = format (0, "cli listener %s", s->config);
 
       clib_file_add (fm, &template);
     }
@@ -3111,12 +3117,55 @@ done:
 
 /* *INDENT-OFF* */
 VLIB_CLI_COMMAND (cli_unix_show_errors, static) = {
-  .path = "show unix-errors",
+  .path = "show unix errors",
   .short_help = "Show Unix system call error history",
   .function = unix_show_errors,
 };
 /* *INDENT-ON* */
 
+/** CLI command to show various unix error statistics. */
+static clib_error_t *
+unix_show_files (vlib_main_t * vm,
+                unformat_input_t * input, vlib_cli_command_t * cmd)
+{
+  clib_error_t *error = 0;
+  clib_file_main_t *fm = &file_main;
+  clib_file_t *f;
+  char path[PATH_MAX];
+  u8 *s = 0;
+
+  vlib_cli_output (vm, "%3s %6s %12s %12s %12s %-32s %s", "FD", "Thread",
+                  "Read", "Write", "Error", "File Name", "Description");
+
+  /* *INDENT-OFF* */
+  pool_foreach (f, fm->file_pool,(
+   {
+      int rv;
+      s = format (s, "/proc/self/fd/%d%c", f->file_descriptor, 0);
+      rv = readlink((char *) s, path, PATH_MAX - 1);
+
+      path[rv < 0 ? 0 : rv] = 0;
+
+      vlib_cli_output (vm, "%3d %6d %12d %12d %12d %-32s %v",
+                      f->file_descriptor, f->polling_thread_index,
+                      f->read_events, f->write_events, f->error_events,
+                      path, f->description);
+      vec_reset_length (s);
+    }));
+  /* *INDENT-ON* */
+  vec_free (s);
+
+  return error;
+}
+
+/* *INDENT-OFF* */
+VLIB_CLI_COMMAND (cli_unix_show_files, static) = {
+  .path = "show unix files",
+  .short_help = "Show Unix files in use",
+  .function = unix_show_files,
+};
+/* *INDENT-ON* */
+
 /** CLI command to show session command history. */
 static clib_error_t *
 unix_cli_show_history (vlib_main_t * vm,

diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c
index ecf659b..0c29844 100644 (file)
--- a/src/vlib/unix/input.c
+++ b/src/vlib/unix/input.c
@@ -40,6 +40,7 @@
 #include <vlib/vlib.h>
 #include <vlib/unix/unix.h>
 #include <signal.h>
+#include <unistd.h>
 #include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
 
 /* FIXME autoconf */
@@ -53,23 +54,23 @@ typedef struct
 {
   int epoll_fd;
   struct epoll_event *epoll_events;
+  int n_epoll_fds;
 
   /* Statistics. */
   u64 epoll_files_ready;
   u64 epoll_waits;
 } linux_epoll_main_t;
 
-static linux_epoll_main_t linux_epoll_main;
+static linux_epoll_main_t *linux_epoll_mains = 0;
 
 static void
 linux_epoll_file_update (clib_file_t * f, clib_file_update_type_t update_type)
 {
   clib_file_main_t *fm = &file_main;
-  linux_epoll_main_t *em = &linux_epoll_main;
-  struct epoll_event e;
-  int op;
-
-  memset (&e, 0, sizeof (e));
+  linux_epoll_main_t *em = vec_elt_at_index (linux_epoll_mains,
+                                            f->polling_thread_index);
+  struct epoll_event e = { 0 };
+  int op, add_del = 0;
 
   e.events = EPOLLIN;
   if (f->flags & UNIX_FILE_DATA_AVAILABLE_TO_WRITE)
@@ -84,6 +85,7 @@ linux_epoll_file_update (clib_file_t * f, clib_file_update_type_t update_type)
     {
     case UNIX_FILE_UPDATE_ADD:
       op = EPOLL_CTL_ADD;
+      add_del = 1;
       break;
 
     case UNIX_FILE_UPDATE_MODIFY:
@@ -92,6 +94,7 @@ linux_epoll_file_update (clib_file_t * f, clib_file_update_type_t update_type)
 
     case UNIX_FILE_UPDATE_DELETE:
       op = EPOLL_CTL_DEL;
+      add_del = -1;
       break;
 
     default:
@@ -99,19 +102,43 @@ linux_epoll_file_update (clib_file_t * f, clib_file_update_type_t update_type)
       return;
     }
 
+  /* worker threads open epoll fd only if needed */
+  if (update_type == UNIX_FILE_UPDATE_ADD && em->epoll_fd == -1)
+    {
+      em->epoll_fd = epoll_create (1);
+      if (em->epoll_fd < 0)
+       {
+         clib_unix_warning ("epoll_create");
+         return;
+       }
+      em->n_epoll_fds = 0;
+    }
+
   if (epoll_ctl (em->epoll_fd, op, f->file_descriptor, &e) < 0)
-    clib_unix_warning ("epoll_ctl");
+    {
+      clib_unix_warning ("epoll_ctl");
+      return;
+    }
+
+  em->n_epoll_fds += add_del;
+
+  if (em->n_epoll_fds == 0)
+    {
+      close (em->epoll_fd);
+      em->epoll_fd = -1;
+    }
 }
 
-static uword
-linux_epoll_input (vlib_main_t * vm,
-                  vlib_node_runtime_t * node, vlib_frame_t * frame)
+static_always_inline uword
+linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
+                         vlib_frame_t * frame, u32 thread_index)
 {
   unix_main_t *um = &unix_main;
   clib_file_main_t *fm = &file_main;
-  linux_epoll_main_t *em = &linux_epoll_main;
+  linux_epoll_main_t *em = vec_elt_at_index (linux_epoll_mains, thread_index);
   struct epoll_event *e;
   int n_fds_ready;
+  int is_main = (thread_index == 0);
 
   {
     vlib_node_main_t *nm = &vm->node_main;
@@ -121,7 +148,7 @@ linux_epoll_input (vlib_main_t * vm,
     f64 vector_rate = vlib_last_vectors_per_main_loop (vm);
 
     /* If we're not working very hard, decide how long to sleep */
-    if (vector_rate < 2 && vm->api_queue_nonempty == 0
+    if (is_main && vector_rate < 2 && vm->api_queue_nonempty == 0
        && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
       {
        ticks_until_expiration = TW (tw_timer_first_expires_in_ticks)
@@ -148,6 +175,13 @@ linux_epoll_input (vlib_main_t * vm,
          }
        node->input_main_loops_per_call = 0;
       }
+    else if (is_main == 0 && vector_rate < 2 &&
+            nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
+      {
+       timeout = 10e-3;
+       timeout_ms = max_timeout_ms;
+       node->input_main_loops_per_call = 0;
+      }
     else                       /* busy */
       {
        /* Don't come back for a respectable number of dispatch cycles */
@@ -155,21 +189,28 @@ linux_epoll_input (vlib_main_t * vm,
       }
 
     /* Allow any signal to wakeup our sleep. */
-    {
-      static sigset_t unblock_all_signals;
-      n_fds_ready = epoll_pwait (em->epoll_fd,
-                                em->epoll_events,
-                                vec_len (em->epoll_events),
-                                timeout_ms, &unblock_all_signals);
-
-      /* This kludge is necessary to run over absurdly old kernels */
-      if (n_fds_ready < 0 && errno == ENOSYS)
-       {
-         n_fds_ready = epoll_wait (em->epoll_fd,
-                                   em->epoll_events,
-                                   vec_len (em->epoll_events), timeout_ms);
-       }
-    }
+    if (is_main || em->epoll_fd != -1)
+      {
+       static sigset_t unblock_all_signals;
+       n_fds_ready = epoll_pwait (em->epoll_fd,
+                                  em->epoll_events,
+                                  vec_len (em->epoll_events),
+                                  timeout_ms, &unblock_all_signals);
+
+       /* This kludge is necessary to run over absurdly old kernels */
+       if (n_fds_ready < 0 && errno == ENOSYS)
+         {
+           n_fds_ready = epoll_wait (em->epoll_fd,
+                                     em->epoll_events,
+                                     vec_len (em->epoll_events), timeout_ms);
+         }
+      }
+    else
+      {
+       if (timeout_ms)
+         usleep (timeout_ms * 1000);
+       return 0;
+      }
   }
 
   if (n_fds_ready < 0)
@@ -196,11 +237,13 @@ linux_epoll_input (vlib_main_t * vm,
          if (e->events & EPOLLIN)
            {
              errors[n_errors] = f->read_function (f);
+             f->read_events++;
              n_errors += errors[n_errors] != 0;
            }
          if (e->events & EPOLLOUT)
            {
              errors[n_errors] = f->write_function (f);
+             f->write_events++;
              n_errors += errors[n_errors] != 0;
            }
        }
@@ -209,6 +252,7 @@ linux_epoll_input (vlib_main_t * vm,
          if (f->error_function)
            {
              errors[n_errors] = f->error_function (f);
+             f->error_events++;
              n_errors += errors[n_errors] != 0;
            }
          else
@@ -225,6 +269,18 @@ linux_epoll_input (vlib_main_t * vm,
   return 0;
 }
 
+static uword
+linux_epoll_input (vlib_main_t * vm,
+                  vlib_node_runtime_t * node, vlib_frame_t * frame)
+{
+  u32 thread_index = vlib_get_thread_index ();
+
+  if (thread_index == 0)
+    return linux_epoll_input_inline (vm, node, frame, 0);
+  else
+    return linux_epoll_input_inline (vm, node, frame, thread_index);
+}
+
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (linux_epoll_input_node,static) = {
   .function = linux_epoll_input,
@@ -236,15 +292,28 @@ VLIB_REGISTER_NODE (linux_epoll_input_node,static) = {
 clib_error_t *
 linux_epoll_input_init (vlib_main_t * vm)
 {
-  linux_epoll_main_t *em = &linux_epoll_main;
+  linux_epoll_main_t *em;
   clib_file_main_t *fm = &file_main;
+  vlib_thread_main_t *tm = vlib_get_thread_main ();
+
 
-  /* Allocate some events. */
-  vec_resize (em->epoll_events, VLIB_FRAME_SIZE);
+  vec_validate_aligned (linux_epoll_mains, tm->n_vlib_mains,
+                       CLIB_CACHE_LINE_BYTES);
 
-  em->epoll_fd = epoll_create (vec_len (em->epoll_events));
-  if (em->epoll_fd < 0)
-    return clib_error_return_unix (0, "epoll_create");
+  vec_foreach (em, linux_epoll_mains)
+  {
+    /* Allocate some events. */
+    vec_resize (em->epoll_events, VLIB_FRAME_SIZE);
+
+    if (linux_epoll_mains == em)
+      {
+       em->epoll_fd = epoll_create (1);
+       if (em->epoll_fd < 0)
+         return clib_error_return_unix (0, "epoll_create");
+      }
+    else
+      em->epoll_fd = -1;
+  }
 
   fm->file_update = linux_epoll_file_update;
 

diff --git a/src/vnet/devices/af_packet/af_packet.c b/src/vnet/devices/af_packet/af_packet.c
index c51fded..2a13510 100644 (file)
--- a/src/vnet/devices/af_packet/af_packet.c
+++ b/src/vnet/devices/af_packet/af_packet.c
@@ -288,6 +288,8 @@ af_packet_create_if (vlib_main_t * vm, u8 * host_if_name, u8 * hw_addr_set,
     template.file_descriptor = fd;
     template.private_data = if_index;
     template.flags = UNIX_FILE_EVENT_EDGE_TRIGGERED;
+    template.description = format (0, "%U", format_af_packet_device_name,
+                                  if_index);
     apif->clib_file_index = clib_file_add (&file_main, &template);
   }
 

diff --git a/src/vnet/devices/af_packet/af_packet.h b/src/vnet/devices/af_packet/af_packet.h
index b095e7f..18822f8 100644 (file)
--- a/src/vnet/devices/af_packet/af_packet.h
+++ b/src/vnet/devices/af_packet/af_packet.h
@@ -66,6 +66,8 @@ int af_packet_delete_if (vlib_main_t * vm, u8 * host_if_name);
 int af_packet_set_l4_cksum_offload (vlib_main_t * vm, u32 sw_if_index,
                                    u8 set);
 
+format_function_t format_af_packet_device_name;
+
 /*
  * fd.io coding-style-patch-verification: ON
  *

diff --git a/src/vnet/devices/af_packet/device.c b/src/vnet/devices/af_packet/device.c
index b9492b3..2c2489b 100644 (file)
--- a/src/vnet/devices/af_packet/device.c
+++ b/src/vnet/devices/af_packet/device.c
@@ -51,7 +51,7 @@ static char *af_packet_tx_func_error_strings[] = {
 };
 
 
-static u8 *
+u8 *
 format_af_packet_device_name (u8 * s, va_list * args)
 {
   u32 i = va_arg (*args, u32);

diff --git a/src/vnet/devices/virtio/virtio.c b/src/vnet/devices/virtio/virtio.c
index 02ded3f..aa9db3a 100644 (file)
--- a/src/vnet/devices/virtio/virtio.c
+++ b/src/vnet/devices/virtio/virtio.c
@@ -109,6 +109,8 @@ virtio_vring_init (vlib_main_t * vm, virtio_if_t * vif, u16 idx, u16 sz)
   t.read_function = call_read_ready;
   t.file_descriptor = vring->call_fd;
   t.private_data = vif->dev_instance << 16 | idx;
+  t.description = format (0, "%U vring %u", format_virtio_device_name,
+                         vif->dev_instance, idx);
   vring->call_file_index = clib_file_add (&file_main, &t);
 
   state.index = idx;

diff --git a/src/vnet/devices/virtio/virtio.h b/src/vnet/devices/virtio/virtio.h
index 90eeb53..7ffb2ea 100644 (file)
--- a/src/vnet/devices/virtio/virtio.h
+++ b/src/vnet/devices/virtio/virtio.h
@@ -132,6 +132,8 @@ clib_error_t *virtio_vring_free (vlib_main_t * vm, virtio_if_t * vif,
                                 u32 idx);
 extern void virtio_free_used_desc (vlib_main_t * vm, virtio_vring_t * vring);
 
+format_function_t format_virtio_device_name;
+
 #endif /* _VNET_DEVICES_VIRTIO_VIRTIO_H_ */
 
 /*

diff --git a/src/vnet/ip/punt.c b/src/vnet/ip/punt.c
index 4a027bf..f24a43f 100644 (file)
--- a/src/vnet/ip/punt.c
+++ b/src/vnet/ip/punt.c
@@ -890,6 +890,7 @@ punt_config (vlib_main_t * vm, unformat_input_t * input)
   clib_file_t template = { 0 };
   template.read_function = punt_socket_read_ready;
   template.file_descriptor = pm->socket_fd;
+  template.description = format (0, "%s", socket_path);
   pm->clib_file_index = clib_file_add (fm, &template);
 
   pm->is_configured = true;

diff --git a/src/vppinfra/file.h b/src/vppinfra/file.h
index f934972..b5a0507 100644 (file)
--- a/src/vppinfra/file.h
+++ b/src/vppinfra/file.h
@@ -57,11 +57,22 @@ typedef struct clib_file
 #define UNIX_FILE_DATA_AVAILABLE_TO_WRITE (1 << 0)
 #define UNIX_FILE_EVENT_EDGE_TRIGGERED   (1 << 1)
 
+  /* polling thread index */
+  u32 polling_thread_index;
+
   /* Data available for function's use. */
   uword private_data;
 
   /* Functions to be called when read/write data becomes ready. */
   clib_file_function_t *read_function, *write_function, *error_function;
+
+  /* Description */
+  u8 *description;
+
+  /* Stats */
+  u64 read_events;
+  u64 write_events;
+  u64 error_events;
 } clib_file_t;
 
 typedef enum
@@ -87,6 +98,9 @@ clib_file_add (clib_file_main_t * um, clib_file_t * template)
   clib_file_t *f;
   pool_get (um->file_pool, f);
   f[0] = template[0];
+  f->read_events = 0;
+  f->write_events = 0;
+  f->error_events = 0;
   um->file_update (f, UNIX_FILE_UPDATE_ADD);
   return f - um->file_pool;
 }
@@ -97,6 +111,7 @@ clib_file_del (clib_file_main_t * um, clib_file_t * f)
   um->file_update (f, UNIX_FILE_UPDATE_DELETE);
   close (f->file_descriptor);
   f->file_descriptor = ~0;
+  vec_free (f->description);
   pool_put (um->file_pool, f);
 }
 
@@ -108,6 +123,16 @@ clib_file_del_by_index (clib_file_main_t * um, uword index)
   clib_file_del (um, uf);
 }
 
+always_inline void
+clib_file_set_polling_thread (clib_file_main_t * um, uword index,
+                             u32 thread_index)
+{
+  clib_file_t *f = pool_elt_at_index (um->file_pool, index);
+  um->file_update (f, UNIX_FILE_UPDATE_DELETE);
+  f->polling_thread_index = thread_index;
+  um->file_update (f, UNIX_FILE_UPDATE_ADD);
+}
+
 always_inline uword
 clib_file_set_data_available_to_write (clib_file_main_t * um,
                                       u32 clib_file_index,