From 940f78fcf4e276cadef3f163d0493b74fc49755c Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Fri, 30 Nov 2018 12:11:20 -0800 Subject: [PATCH] vcl: handle worker process exit Change-Id: Ife05d25fd736ae3064f01e974e5aecc5b48de924 Signed-off-by: Florin Coras --- src/vcl/ldp.c | 5 +---- src/vcl/vcl_private.c | 13 ++++++++----- src/vcl/vcl_private.h | 2 +- src/vcl/vppcom.c | 36 +++++++++++++++++++++++++++++++----- src/vlibmemory/memory_client.c | 21 ++++++++++++++------- src/vlibmemory/memory_client.h | 1 + 6 files changed, 56 insertions(+), 22 deletions(-) diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c index 915d1caa40c..453ddeb3340 100644 --- a/src/vcl/ldp.c +++ b/src/vcl/ldp.c @@ -3556,10 +3556,7 @@ ldp_destructor (void) { swrap_destructor (); if (ldp->init) - { - vppcom_app_destroy (); - ldp->init = 0; - } + ldp->init = 0; /* Don't use clib_warning() here because that calls writev() * which will call ldp_init(). diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c index 76ae0e7719f..673b91d4576 100644 --- a/src/vcl/vcl_private.c +++ b/src/vcl/vcl_private.c @@ -221,13 +221,15 @@ vcl_worker_free (vcl_worker_t * wrk) } void -vcl_worker_cleanup (void) +vcl_worker_cleanup (u8 notify_vpp) { vcl_worker_t *wrk = vcl_worker_get_current (); clib_spinlock_lock (&vcm->workers_lock); - vcl_send_app_worker_add_del (0 /* is_add */ ); - close (wrk->mqs_epfd); + if (notify_vpp) + vcl_send_app_worker_add_del (0 /* is_add */ ); + if (wrk->mqs_epfd > 0) + close (wrk->mqs_epfd); hash_free (wrk->session_index_by_vpp_handles); hash_free (wrk->ct_registration_by_mq); clib_spinlock_free (&wrk->ct_registration_lock); @@ -236,13 +238,14 @@ vcl_worker_cleanup (void) vcl_set_worker_index (~0); vcl_worker_free (wrk); clib_spinlock_unlock (&vcm->workers_lock); - VDBG (0, "cleaned up worker %u", wrk->wrk_index); } static void vcl_worker_cleanup_cb (void *arg) { - vcl_worker_cleanup (); + u32 wrk_index = vcl_get_worker_index (); + vcl_worker_cleanup (1 /* notify vpp */ ); + VDBG (0, "cleaned up worker %u", wrk_index); } vcl_worker_t * diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 6a289d640e0..7d6af6d6599 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -504,7 +504,7 @@ int vcl_mq_epoll_add_evfd (vcl_worker_t * wrk, svm_msg_q_t * mq); int vcl_mq_epoll_del_evfd (vcl_worker_t * wrk, u32 mqc_index); vcl_worker_t *vcl_worker_alloc_and_init (void); -void vcl_worker_cleanup (void); +void vcl_worker_cleanup (u8 notify_vpp); int vcl_worker_register_with_vpp (void); int vcl_worker_set_bapi (void); void vcl_worker_share_sessions (u32 parent_wrk_index); diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index db767e9e713..1b619cf3f21 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -700,12 +700,12 @@ vcl_app_fork_child_handler (void) u8 *child_name; int rv, parent_wrk; - VDBG (0, "initializing forked child"); + parent_wrk = vcl_get_worker_index (); + VDBG (0, "initializing forked child with parent wrk %u", parent_wrk); /* * Allocate worker */ - parent_wrk = vcl_get_worker_index (); vcl_set_worker_index (~0); if (!vcl_worker_alloc_and_init ()) VERR ("couldn't allocate new worker"); @@ -744,6 +744,27 @@ vcl_app_fork_parent_handler (void) ; } +/** + * Handle app exit + * + * Notify vpp of the disconnect and mark the worker as free. If we're the + * last worker, do a full cleanup otherwise, since we're probably a forked + * child, avoid syscalls as much as possible. We might've lost privileges. + */ +void +vppcom_app_exit (void) +{ + if (!pool_elts (vcm->workers)) + return; + + vcl_worker_cleanup (1 /* notify vpp */ ); + vcl_elog_stop (vcm); + if (vec_len (vcm->workers) == 1) + vl_client_disconnect_from_vlib (); + else + vl_client_send_disconnect (); +} + /* * VPPCOM Public API functions */ @@ -774,6 +795,7 @@ vppcom_app_create (char *app_name) clib_rwlock_init (&vcm->segment_table_lock); pthread_atfork (NULL, vcl_app_fork_parent_handler, vcl_app_fork_child_handler); + atexit (vppcom_app_exit); /* Allocate default worker */ vcl_worker_alloc_and_init (); @@ -816,9 +838,12 @@ vppcom_app_destroy (void) int rv; f64 orig_app_timeout; + if (!pool_elts (vcm->workers)) + return; + vcl_evt (VCL_EVT_DETACH, vcm); - if (vec_len (vcm->workers) == 1) + if (pool_elts (vcm->workers) == 1) { vppcom_app_send_detach (); orig_app_timeout = vcm->cfg.app_timeout; @@ -828,15 +853,16 @@ vppcom_app_destroy (void) if (PREDICT_FALSE (rv)) VDBG (0, "application detach timed out! returning %d (%s)", rv, vppcom_retval_str (rv)); + vec_free (vcm->app_name); + vcl_worker_cleanup (0 /* notify vpp */ ); } else { - vcl_worker_cleanup (); + vcl_worker_cleanup (1 /* notify vpp */ ); } vcl_elog_stop (vcm); vl_client_disconnect_from_vlib (); - vec_free (vcm->app_name); } int diff --git a/src/vlibmemory/memory_client.c b/src/vlibmemory/memory_client.c index 239cbfee36f..79344a6af68 100644 --- a/src/vlibmemory/memory_client.c +++ b/src/vlibmemory/memory_client.c @@ -270,22 +270,17 @@ vl_api_memclnt_delete_reply_t_handler (vl_api_memclnt_delete_reply_t * mp) am->vl_input_queue = 0; } -int -vl_client_disconnect (void) +void +vl_client_send_disconnect (void) { vl_api_memclnt_delete_t *mp; - vl_api_memclnt_delete_reply_t *rp; - svm_queue_t *vl_input_queue; vl_shmem_hdr_t *shmem_hdr; - time_t begin; api_main_t *am = &api_main; ASSERT (am->vlib_rp); shmem_hdr = am->shmem_hdr; ASSERT (shmem_hdr && shmem_hdr->vl_input_queue); - vl_input_queue = am->vl_input_queue; - mp = vl_msg_api_alloc (sizeof (vl_api_memclnt_delete_t)); clib_memset (mp, 0, sizeof (*mp)); mp->_vl_msg_id = ntohs (VL_API_MEMCLNT_DELETE); @@ -293,6 +288,18 @@ vl_client_disconnect (void) mp->handle = (uword) am->my_registration; vl_msg_api_send_shmem (shmem_hdr->vl_input_queue, (u8 *) & mp); +} + +int +vl_client_disconnect (void) +{ + vl_api_memclnt_delete_reply_t *rp; + svm_queue_t *vl_input_queue; + api_main_t *am = &api_main; + time_t begin; + + vl_input_queue = am->vl_input_queue; + vl_client_send_disconnect (); /* * Have to be careful here, in case the client is disconnecting diff --git a/src/vlibmemory/memory_client.h b/src/vlibmemory/memory_client.h index f2898f30b23..eb3431a51e6 100644 --- a/src/vlibmemory/memory_client.h +++ b/src/vlibmemory/memory_client.h @@ -21,6 +21,7 @@ #include int vl_client_connect (const char *name, int ctx_quota, int input_queue_size); +void vl_client_send_disconnect (void); int vl_client_disconnect (void); int vl_client_api_map (const char *region_name); void vl_client_api_unmap (void); -- 2.16.6