From: Florin Coras Date: Fri, 30 Nov 2018 01:02:29 +0000 (-0800) Subject: vcl: wait for segments with segment handle X-Git-Tag: v19.04-rc0~261 X-Git-Url: https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commitdiff_plain;h=d85de68;hp=e4f849c137101871b1caa0d959c7ea794e7d77f7 vcl: wait for segments with segment handle Instead of waiting for notification from binary api. Change-Id: I5ecab857d6bcdbed62d6bb06709570c4cf6b19ea Signed-off-by: Florin Coras --- diff --git a/src/vcl/vcl_bapi.c b/src/vcl/vcl_bapi.c index 7f3a5741af3..c6e7fdf369d 100644 --- a/src/vcl/vcl_bapi.c +++ b/src/vcl/vcl_bapi.c @@ -62,7 +62,8 @@ static void } static int -ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd) +vcl_segment_attach (u64 segment_handle, char *name, ssvm_segment_type_t type, + int fd) { svm_fifo_segment_create_args_t _a, *a = &_a; int rv; @@ -79,15 +80,37 @@ ssvm_segment_attach (char *name, ssvm_segment_type_t type, int fd) clib_warning ("svm_fifo_segment_attach ('%s') failed", name); return rv; } + vcl_segment_table_add (segment_handle, a->new_segment_indices[0]); vec_reset_length (a->new_segment_indices); return 0; } +static void +vcl_segment_detach (u64 segment_handle) +{ + svm_fifo_segment_private_t *segment; + u32 segment_index; + + segment_index = vcl_segment_table_lookup (segment_handle); + if (segment_index == (u32) ~ 0) + return; + segment = svm_fifo_segment_get_segment (segment_index); + svm_fifo_segment_delete (segment); + vcl_segment_table_del (segment_handle); +} + +static u64 +vcl_vpp_worker_segment_handle (u32 wrk_index) +{ + return (VCL_INVALID_SEGMENT_HANDLE - wrk_index - 1); +} + static void vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * mp) { vcl_worker_t *wrk = vcl_worker_get (0); + u64 segment_handle; u32 n_fds = 0; int *fds = 0; @@ -100,19 +123,27 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address, svm_msg_q_t *); + segment_handle = clib_net_to_host_u64 (mp->segment_handle); + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + return; + } + if (mp->n_fds) { vec_validate (fds, mp->n_fds); vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5); if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) - if (ssvm_segment_attach ("vpp-mq-seg", SSVM_SEGMENT_MEMFD, - fds[n_fds++])) + if (vcl_segment_attach (vcl_vpp_worker_segment_handle (0), + "vpp-mq-seg", SSVM_SEGMENT_MEMFD, + fds[n_fds++])) return; if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) - if (ssvm_segment_attach ((char *) mp->segment_name, - SSVM_SEGMENT_MEMFD, fds[n_fds++])) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds++])) return; if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) @@ -126,8 +157,8 @@ vl_api_application_attach_reply_t_handler (vl_api_application_attach_reply_t * } else { - if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM, - -1)) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_SHM, -1)) return; } @@ -140,6 +171,7 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * mp) { int n_fds = 0, *fds = 0; + u64 segment_handle; vcl_worker_t *wrk; u32 wrk_index; @@ -159,19 +191,27 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * wrk->app_event_queue = uword_to_pointer (mp->app_event_queue_address, svm_msg_q_t *); + segment_handle = clib_net_to_host_u64 (mp->segment_handle); + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + return; + } + if (mp->n_fds) { vec_validate (fds, mp->n_fds); vl_socket_client_recv_fd_msg (fds, mp->n_fds, 5); if (mp->fd_flags & SESSION_FD_F_VPP_MQ_SEGMENT) - if (ssvm_segment_attach ("vpp-worker-seg", SSVM_SEGMENT_MEMFD, - fds[n_fds++])) + if (vcl_segment_attach + (vcl_vpp_worker_segment_handle (wrk->wrk_index), "vpp-worker-seg", + SSVM_SEGMENT_MEMFD, fds[n_fds++])) goto failed; if (mp->fd_flags & SESSION_FD_F_MEMFD_SEGMENT) - if (ssvm_segment_attach ((char *) mp->segment_name, - SSVM_SEGMENT_MEMFD, fds[n_fds++])) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_MEMFD, fds[n_fds++])) goto failed; if (mp->fd_flags & SESSION_FD_F_MQ_EVENTFD) @@ -185,8 +225,8 @@ vl_api_app_worker_add_del_reply_t_handler (vl_api_app_worker_add_del_reply_t * } else { - if (ssvm_segment_attach ((char *) mp->segment_name, SSVM_SEGMENT_SHM, - -1)) + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + SSVM_SEGMENT_SHM, -1)) goto failed; } vcm->app_state = STATE_APP_READY; @@ -212,18 +252,24 @@ static void vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp) { ssvm_segment_type_t seg_type = SSVM_SEGMENT_SHM; + u64 segment_handle; int fd = -1; - vcm->mounting_segment = 1; - if (mp->fd_flags) { vl_socket_client_recv_fd_msg (&fd, 1, 5); seg_type = SSVM_SEGMENT_MEMFD; } - if (PREDICT_FALSE (ssvm_segment_attach ((char *) mp->segment_name, - seg_type, fd))) + segment_handle = clib_net_to_host_u64 (mp->segment_handle); + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + { + clib_warning ("invalid segment handle"); + return; + } + + if (vcl_segment_attach (segment_handle, (char *) mp->segment_name, + seg_type, fd)) { clib_warning ("VCL<%d>: svm_fifo_segment_attach ('%s') failed", getpid (), mp->segment_name); @@ -232,20 +278,14 @@ vl_api_map_another_segment_t_handler (vl_api_map_another_segment_t * mp) VDBG (1, "VCL<%d>: mapped new segment '%s' size %d", getpid (), mp->segment_name, mp->segment_size); - vcm->mounting_segment = 0; } static void vl_api_unmap_segment_t_handler (vl_api_unmap_segment_t * mp) { - -/* - * XXX Need segment_name to session_id hash, - * XXX - have sessionID by handle hash currently - */ - - VDBG (1, "Unmapped segment '%s'", - clib_net_to_host_u64 (mp->segment_handle)); + u64 segment_handle = clib_net_to_host_u64 (mp->segment_handle); + vcl_segment_detach (segment_handle); + VDBG (1, "Unmapped segment: %d", segment_handle); } static void diff --git a/src/vcl/vcl_private.c b/src/vcl/vcl_private.c index 86dccfe1cc9..76ae0e7719f 100644 --- a/src/vcl/vcl_private.c +++ b/src/vcl/vcl_private.c @@ -442,6 +442,36 @@ vcl_session_get_refcnt (vcl_session_t * s) return 0; } +void +vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index) +{ + clib_rwlock_writer_lock (&vcm->segment_table_lock); + hash_set (vcm->segment_table, segment_handle, svm_segment_index); + clib_rwlock_writer_unlock (&vcm->segment_table_lock); +} + +u32 +vcl_segment_table_lookup (u64 segment_handle) +{ + uword *seg_indexp; + + clib_rwlock_reader_lock (&vcm->segment_table_lock); + seg_indexp = hash_get (vcm->segment_table, segment_handle); + clib_rwlock_reader_unlock (&vcm->segment_table_lock); + + if (!seg_indexp) + return VCL_INVALID_SEGMENT_INDEX; + return ((u32) * seg_indexp); +} + +void +vcl_segment_table_del (u64 segment_handle) +{ + clib_rwlock_writer_lock (&vcm->segment_table_lock); + hash_unset (vcm->segment_table, segment_handle); + clib_rwlock_writer_unlock (&vcm->segment_table_lock); +} + /* * fd.io coding-style-patch-verification: ON * diff --git a/src/vcl/vcl_private.h b/src/vcl/vcl_private.h index 0420322d3f4..6ca0471e965 100644 --- a/src/vcl/vcl_private.h +++ b/src/vcl/vcl_private.h @@ -309,9 +309,6 @@ typedef struct vppcom_main_t_ /** VCL configuration */ vppcom_cfg_t cfg; - /** Flag indicating that a new segment is being mounted */ - volatile u32 mounting_segment; - volatile u32 forking; /** Workers */ @@ -320,7 +317,15 @@ typedef struct vppcom_main_t_ /** Lock to protect worker registrations */ clib_spinlock_t workers_lock; + /** Pool of shared sessions */ vcl_shared_session_t *shared_sessions; + + /** Lock to protect segment hash table */ + clib_rwlock_t segment_table_lock; + + /** Mapped segments table */ + uword *segment_table; + #ifdef VCL_ELOG /* VPP Event-logger */ elog_main_t elog_main; @@ -335,6 +340,8 @@ typedef struct vppcom_main_t_ extern vppcom_main_t *vcm; #define VCL_INVALID_SESSION_INDEX ((u32)~0) +#define VCL_INVALID_SEGMENT_INDEX ((u32)~0) +#define VCL_INVALID_SEGMENT_HANDLE ((u64)~0) static inline vcl_session_t * vcl_session_alloc (vcl_worker_t * wrk) @@ -502,6 +509,10 @@ void vcl_worker_share_sessions (u32 parent_wrk_index); int vcl_worker_unshare_session (vcl_worker_t * wrk, vcl_session_t * s); int vcl_session_get_refcnt (vcl_session_t * s); +void vcl_segment_table_add (u64 segment_handle, u32 svm_segment_index); +u32 vcl_segment_table_lookup (u64 segment_handle); +void vcl_segment_table_del (u64 segment_handle); + static inline vcl_worker_t * vcl_worker_get (u32 wrk_index) { diff --git a/src/vcl/vppcom.c b/src/vcl/vppcom.c index 195e6cbc602..34b663b9831 100644 --- a/src/vcl/vppcom.c +++ b/src/vcl/vppcom.c @@ -22,46 +22,26 @@ __thread uword __vcl_worker_index = ~0; -static u8 not_ready; -void -sigsegv_signal (int signum) +static int +vcl_wait_for_segment (u64 segment_handle) { - not_ready = 1; -} + vcl_worker_t *wrk = vcl_worker_get_current (); + u32 wait_for_seconds = 10, segment_index; + f64 timeout; -static void -vcl_wait_for_memory (void *mem) -{ - u8 __clib_unused test; - if (vcm->mounting_segment) - { - while (vcm->mounting_segment) - ; - return; - } - if (1 || vcm->debug) - { - usleep (1e5); - return; - } - if (signal (SIGSEGV, sigsegv_signal)) - { - perror ("signal()"); - return; - } - not_ready = 0; + if (segment_handle == VCL_INVALID_SEGMENT_HANDLE) + return 1; -again: - test = *(u8 *) mem; - if (not_ready) + timeout = clib_time_now (&wrk->clib_time) + wait_for_seconds; + while (clib_time_now (&wrk->clib_time) < timeout) { - not_ready = 0; - usleep (1); - goto again; + segment_index = vcl_segment_table_lookup (segment_handle); + if (segment_index != VCL_INVALID_SEGMENT_INDEX) + return 0; + usleep (10); } - - signal (SIGSEGV, SIG_DFL); + return 1; } const char * @@ -281,7 +261,12 @@ vcl_session_accepted_handler (vcl_worker_t * wrk, session_accepted_msg_t * mp) svm_msg_q_t *); session->our_evt_q = uword_to_pointer (mp->server_event_queue_address, svm_msg_q_t *); - vcl_wait_for_memory (session->vpp_evt_q); + if (vcl_wait_for_segment (mp->segment_handle)) + { + clib_warning ("segment for session %u couldn't be mounted!", + session->session_index); + return VCL_INVALID_SESSION_INDEX; + } rx_fifo->master_session_index = session->session_index; tx_fifo->master_session_index = session->session_index; rx_fifo->master_thread_index = vcl_get_worker_index (); @@ -360,7 +345,13 @@ vcl_session_connected_handler (vcl_worker_t * wrk, rx_fifo = uword_to_pointer (mp->server_rx_fifo, svm_fifo_t *); tx_fifo = uword_to_pointer (mp->server_tx_fifo, svm_fifo_t *); - vcl_wait_for_memory (rx_fifo); + if (vcl_wait_for_segment (mp->segment_handle)) + { + clib_warning ("segment for session %u couldn't be mounted!", + session->session_index); + return VCL_INVALID_SESSION_INDEX; + } + rx_fifo->client_session_index = session_index; tx_fifo->client_session_index = session_index; rx_fifo->client_thread_index = vcl_get_worker_index (); @@ -436,8 +427,8 @@ vcl_session_bound_handler (vcl_worker_t * wrk, session_bound_msg_t * mp) session = vcl_session_get (wrk, sid); if (mp->retval) { - VDBG (0, "VCL<%d>: ERROR: vpp handle 0x%llx, sid %u: bind failed: %U", - getpid (), mp->handle, sid, format_api_error, ntohl (mp->retval)); + VERR ("vpp handle 0x%llx, sid %u: bind failed: %U", mp->handle, sid, + format_api_error, mp->retval); if (session) { session->session_state = STATE_FAILED; @@ -780,6 +771,7 @@ vppcom_app_create (char *app_name) 20 /* timeout in secs */ ); pool_alloc (vcm->workers, vcl_cfg->max_workers); clib_spinlock_init (&vcm->workers_lock); + clib_rwlock_init (&vcm->segment_table_lock); pthread_atfork (NULL, vcl_app_fork_parent_handler, vcl_app_fork_child_handler); diff --git a/src/vnet/session/application.c b/src/vnet/session/application.c index 69f33290a73..f3628966ae5 100644 --- a/src/vnet/session/application.c +++ b/src/vnet/session/application.c @@ -1591,8 +1591,8 @@ application_local_session_connect (app_worker_t * client_wrk, local_session_t * ll, u32 opaque) { u32 seg_size, evt_q_sz, evt_q_elts, margin = 16 << 10; + u32 round_rx_fifo_sz, round_tx_fifo_sz, sm_index; segment_manager_properties_t *props, *cprops; - u32 round_rx_fifo_sz, round_tx_fifo_sz; int rv, has_transport, seg_index; svm_fifo_segment_private_t *seg; application_t *server, *client; @@ -1654,8 +1654,13 @@ application_local_session_connect (app_worker_t * client_wrk, segment_manager_segment_reader_unlock (sm); goto failed; } + sm_index = segment_manager_index (sm); ls->server_rx_fifo->ct_session_index = ls->session_index; ls->server_tx_fifo->ct_session_index = ls->session_index; + ls->server_rx_fifo->segment_manager = sm_index; + ls->server_tx_fifo->segment_manager = sm_index; + ls->server_rx_fifo->segment_index = seg_index; + ls->server_tx_fifo->segment_index = seg_index; ls->svm_segment_index = seg_index; ls->listener_index = ll->session_index; ls->client_wrk_index = client_wrk->wrk_index; diff --git a/src/vnet/session/application_interface.c b/src/vnet/session/application_interface.c index ace9ed2f868..d254b9bb55a 100644 --- a/src/vnet/session/application_interface.c +++ b/src/vnet/session/application_interface.c @@ -558,6 +558,7 @@ vnet_application_attach (vnet_app_attach_args_t * a) ASSERT (vec_len (fs->ssvm.name) <= 128); a->segment = &fs->ssvm; + a->segment_handle = segment_manager_segment_handle (sm, fs); segment_manager_segment_reader_unlock (sm); vec_free (app_name); diff --git a/src/vnet/session/application_interface.h b/src/vnet/session/application_interface.h index ecd99d8755b..dfb45a78fc9 100644 --- a/src/vnet/session/application_interface.h +++ b/src/vnet/session/application_interface.h @@ -28,6 +28,7 @@ typedef struct _vnet_app_attach_args_t #undef _ ssvm_private_t * segment; svm_msg_q_t *app_evt_q; + u64 segment_handle; } vnet_app_attach_args_t; typedef struct _vnet_app_detach_args_t diff --git a/src/vnet/session/session.api b/src/vnet/session/session.api index 3dc0cc8d360..084678f2518 100644 --- a/src/vnet/session/session.api +++ b/src/vnet/session/session.api @@ -13,7 +13,7 @@ * limitations under the License. */ -option version = "1.4.0"; +option version = "1.5.0"; /** \brief client->vpp, attach application to session layer @param client_index - opaque cookie to identify the sender @@ -45,6 +45,7 @@ option version = "1.4.0"; @param segment_name_length - length of segment name @param segment_name - name of segment client needs to attach to @param app_index - index of the newly created app + @param segment_handle - handle for segment */ define application_attach_reply { u32 context; @@ -56,6 +57,7 @@ define application_attach_reply { u8 segment_name_length; u8 segment_name[128]; u32 app_index; + u64 segment_handle; }; /** \brief Application add TLS certificate diff --git a/src/vnet/session/session_api.c b/src/vnet/session/session_api.c index d37b3c995d0..6ce9e1feac3 100755 --- a/src/vnet/session/session_api.c +++ b/src/vnet/session/session_api.c @@ -825,6 +825,7 @@ done: rmp->app_event_queue_address = pointer_to_uword (a->app_evt_q); rmp->n_fds = n_fds; rmp->fd_flags = fd_flags; + rmp->segment_handle = clib_host_to_net_u64 (a->segment_handle); } })); /* *INDENT-ON* */ diff --git a/test/test_vcl.py b/test/test_vcl.py index 0eacf57c78e..5691604665b 100644 --- a/test/test_vcl.py +++ b/test/test_vcl.py @@ -347,13 +347,13 @@ class VCLThruHostStackNSessionBidirTestCase(VCLTestCase): self.thru_host_stack_setup() if self.vppDebug: - self.client_bi_dir_nsock_timeout = 120 + self.client_bi_dir_nsock_timeout = 20 self.client_bi_dir_nsock_test_args = ["-N", "1000", "-B", "-X", "-I", "2", self.loop0.local_ip4, self.server_port] else: - self.client_bi_dir_nsock_timeout = 90 + self.client_bi_dir_nsock_timeout = 20 self.client_bi_dir_nsock_test_args = ["-N", "1000", "-B", "-X", "-I", "2", self.loop0.local_ip4,