vcl: improve vpp detached handling for mq epoll apps

Apps that rely on epoll and use eventfds for mq signaling can
potentially sleep indefinitely if vcl detaches from vpp and no libc fd
generates events.

To avoid this, at detach time, force creation of a pair of pipes that
constantly generates mq_epfd events to force apps, including ldp, to
request vppcom epoll_waits which retry attaching to vpp.

Type: improvement

Signed-off-by: Florin Coras <[email protected]>
Change-Id: Ie2ac338cc10721829e6ac525ee7d6b812354e9a3

diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c
index 6892688..1adc8cb 100644 (file)
--- a/src/vcl/vcl_private.c
+++ b/src/vcl/vcl_private.c
@@ -188,6 +188,53 @@ vcl_worker_cleanup_cb (void *arg)
   VDBG (0, "cleaned up worker %u", wrk_index);
 }
 
+void
+vcl_worker_detached_start_signal_mq (vcl_worker_t *wrk)
+{
+  /* Generate mq epfd events using pipes to hopefully force
+   * calls into epoll_wait which retries attaching to vpp */
+  if (!wrk->detached_pipefds[0])
+    {
+      if (pipe (wrk->detached_pipefds))
+       {
+         VDBG (0, "failed to add mq eventfd to mq epoll fd");
+         exit (1);
+       }
+    }
+
+  struct epoll_event evt = {};
+  evt.events = EPOLLIN;
+  evt.data.u32 = wrk->detached_pipefds[0];
+  if (epoll_ctl (wrk->mqs_epfd, EPOLL_CTL_ADD, wrk->detached_pipefds[0],
+                &evt) < 0)
+    {
+      VDBG (0, "failed to add mq eventfd to mq epoll fd");
+      exit (1);
+    }
+
+  int sig = 1, __clib_unused rv;
+  rv = write (wrk->detached_pipefds[1], &sig, 1);
+}
+
+void
+vcl_worker_detached_signal_mq (vcl_worker_t *wrk)
+{
+  int buf, __clib_unused rv;
+  rv = read (wrk->detached_pipefds[0], &buf, 1);
+  rv = write (wrk->detached_pipefds[1], &buf, 1);
+}
+
+void
+vcl_worker_detached_stop_signal_mq (vcl_worker_t *wrk)
+{
+  if (epoll_ctl (wrk->mqs_epfd, EPOLL_CTL_DEL, wrk->detached_pipefds[0], 0) <
+      0)
+    {
+      VDBG (0, "failed to del mq eventfd to mq epoll fd");
+      exit (1);
+    }
+}
+
 void
 vcl_worker_detach_sessions (vcl_worker_t *wrk)
 {
@@ -239,6 +286,8 @@ vcl_worker_detach_sessions (vcl_worker_t *wrk)
 
   vec_free (seg_indices);
   hash_free (seg_indices_map);
+
+  vcl_worker_detached_start_signal_mq (wrk);
 }
 
 void

diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h
index c92bb58..72f0a83 100644 (file)
--- a/src/vcl/vcl_private.h
+++ b/src/vcl/vcl_private.h
@@ -325,6 +325,9 @@ typedef struct vcl_worker_
   /* functions to be called pre/post wait if vcl managed by vls */
   vcl_worker_wait_mq_fn pre_wait_fn;
   vcl_worker_wait_mq_fn post_wait_fn;
+
+  /* mq_epfd signal pipes when wrk detached from vpp */
+  int detached_pipefds[2];
 } vcl_worker_t;
 
 STATIC_ASSERT (sizeof (session_disconnected_msg_t) <= 16,
@@ -799,6 +802,9 @@ void vcl_worker_detach_sessions (vcl_worker_t *wrk);
 void vcl_worker_set_wait_mq_fns (vcl_worker_wait_mq_fn pre_wait,
                                 vcl_worker_wait_mq_fn post_wait);
 
+void vcl_worker_detached_start_signal_mq (vcl_worker_t *wrk);
+void vcl_worker_detached_signal_mq (vcl_worker_t *wrk);
+void vcl_worker_detached_stop_signal_mq (vcl_worker_t *wrk);
 /*
  * VCL Binary API
  */

diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c
index 1e9c915..8bc5f42 100644 (file)
--- a/src/vcl/vppcom.c
+++ b/src/vcl/vppcom.c
@@ -1404,6 +1404,8 @@ vcl_api_retry_attach (vcl_worker_t *wrk)
 {
   vcl_session_t *s;
 
+  vcl_worker_detached_signal_mq (wrk);
+
   clib_spinlock_lock (&vcm->workers_lock);
   if (vcl_is_first_reattach_to_execute ())
     {
@@ -1412,6 +1414,7 @@ vcl_api_retry_attach (vcl_worker_t *wrk)
          clib_spinlock_unlock (&vcm->workers_lock);
          return;
        }
+      vcl_worker_detached_stop_signal_mq (wrk);
       vcl_set_reattach_counter ();
       clib_spinlock_unlock (&vcm->workers_lock);
     }