From: Florin Coras Date: Thu, 2 Aug 2018 19:16:03 +0000 (-0700) Subject: socket api: do not delay sending of messages X-Git-Tag: v18.10-rc1~163 X-Git-Url: https://gerrit.fd.io/r/gitweb?p=vpp.git;a=commitdiff_plain;h=8023ad4c41090995c91f529a94519ffaa605d238 socket api: do not delay sending of messages Instead of relying on main epoll loop to send messages, try to send as soon as possible. Change-Id: I27c0b4076f3599ad6e968df4746881a6717d4299 Signed-off-by: Florin Coras --- diff --git a/src/vlibmemory/socket_api.c b/src/vlibmemory/socket_api.c index 51d35792244..afe02d20536 100644 --- a/src/vlibmemory/socket_api.c +++ b/src/vlibmemory/socket_api.c @@ -85,6 +85,8 @@ vl_socket_api_send (vl_api_registration_t * rp, u8 * elem) api_main_t *am = &api_main; msgbuf_t *mb = (msgbuf_t *) (elem - offsetof (msgbuf_t, data)); vl_api_registration_t *sock_rp; + clib_file_main_t *fm = &file_main; + clib_error_t *error; clib_file_t *cf; cf = vl_api_registration_file (rp); @@ -102,10 +104,21 @@ vl_socket_api_send (vl_api_registration_t * rp, u8 * elem) ASSERT (sock_rp); /* Add the msgbuf_t to the output vector */ - vl_socket_add_pending_output_no_flush (cf, sock_rp, (u8 *) mb, - sizeof (*mb)); - /* Send the message */ - vl_socket_add_pending_output (cf, sock_rp, elem, ntohl (mb->data_len)); + vec_add (sock_rp->output_vector, (u8 *) mb, sizeof (*mb)); + + /* Try to send the message and save any error like + * we do in the input epoll loop */ + vec_add (sock_rp->output_vector, elem, ntohl (mb->data_len)); + error = clib_file_write (cf); + unix_save_error (&unix_main, error); + + /* If we didn't finish sending everything, wait for tx space */ + if (vec_len (sock_rp->output_vector) > 0 + && !(cf->flags & UNIX_FILE_DATA_AVAILABLE_TO_WRITE)) + { + cf->flags |= UNIX_FILE_DATA_AVAILABLE_TO_WRITE; + fm->file_update (cf, UNIX_FILE_UPDATE_MODIFY); + } #if CLIB_DEBUG > 1 output_length = sizeof (*mb) + ntohl (mb->data_len); @@ -266,47 +279,6 @@ vl_socket_read_ready (clib_file_t * uf) return 0; } -void -vl_socket_add_pending_output (clib_file_t * uf, - vl_api_registration_t * rp, - u8 * buffer, uword buffer_bytes) -{ - clib_file_main_t *fm = &file_main; - - vec_add (rp->output_vector, buffer, buffer_bytes); - if (vec_len (rp->output_vector) > 0) - { - int skip_update = 0 != (uf->flags & UNIX_FILE_DATA_AVAILABLE_TO_WRITE); - uf->flags |= UNIX_FILE_DATA_AVAILABLE_TO_WRITE; - if (!skip_update) - fm->file_update (uf, UNIX_FILE_UPDATE_MODIFY); - } -} - -void -vl_socket_add_pending_output_no_flush (clib_file_t * uf, - vl_api_registration_t * rp, - u8 * buffer, uword buffer_bytes) -{ - vec_add (rp->output_vector, buffer, buffer_bytes); -} - -static void -socket_del_pending_output (clib_file_t * uf, - vl_api_registration_t * rp, uword n_bytes) -{ - clib_file_main_t *fm = &file_main; - - vec_delete (rp->output_vector, n_bytes, 0); - if (vec_len (rp->output_vector) <= 0) - { - int skip_update = 0 == (uf->flags & UNIX_FILE_DATA_AVAILABLE_TO_WRITE); - uf->flags &= ~UNIX_FILE_DATA_AVAILABLE_TO_WRITE; - if (!skip_update) - fm->file_update (uf, UNIX_FILE_UPDATE_MODIFY); - } -} - clib_error_t * vl_socket_write_ready (clib_file_t * uf) { @@ -317,21 +289,27 @@ vl_socket_write_ready (clib_file_t * uf) rp = pool_elt_at_index (socket_main.registration_pool, uf->private_data); /* Flush output vector. */ - n = write (uf->file_descriptor, - rp->output_vector, vec_len (rp->output_vector)); + n = write (uf->file_descriptor, rp->output_vector, + vec_len (rp->output_vector)); if (n < 0) { #if DEBUG > 2 clib_warning ("write error, close the file...\n"); #endif clib_file_del (fm, uf); - vl_socket_free_registration_index (rp - socket_main.registration_pool); return 0; } - else if (n > 0) - socket_del_pending_output (uf, rp, n); + { + vec_delete (rp->output_vector, n, 0); + if (vec_len (rp->output_vector) <= 0 + && (uf->flags & UNIX_FILE_DATA_AVAILABLE_TO_WRITE)) + { + uf->flags &= ~UNIX_FILE_DATA_AVAILABLE_TO_WRITE; + fm->file_update (uf, UNIX_FILE_UPDATE_MODIFY); + } + } return 0; } @@ -616,19 +594,18 @@ reply: rmp->context = mp->context; rmp->retval = htonl (rv); - vl_api_send_msg (regp, (u8 *) rmp); + /* + * Note: The reply message needs to make it out the back door + * before we send the magic fd message. That's taken care of by + * the send function. + */ + vl_socket_api_send (regp, (u8 *) rmp); if (rv != 0) return; - /* - * We need the reply message to make it out the back door - * before we send the magic fd message so force a flush - */ - cf = vl_api_registration_file (regp); - cf->write_function (cf); - /* Send the magic "here's your sign (aka fd)" socket message */ + cf = vl_api_registration_file (regp); vl_sock_api_send_fd_msg (cf->file_descriptor, &memfd->fd, 1); } diff --git a/src/vlibmemory/socket_api.h b/src/vlibmemory/socket_api.h index 86fd600edbb..1e99550e64b 100644 --- a/src/vlibmemory/socket_api.h +++ b/src/vlibmemory/socket_api.h @@ -65,12 +65,6 @@ extern socket_main_t socket_main; void vl_socket_free_registration_index (u32 pool_index); clib_error_t *vl_socket_read_ready (struct clib_file *uf); -void vl_socket_add_pending_output (struct clib_file *uf, - struct vl_api_registration_ *rp, - u8 * buffer, uword buffer_bytes); -void vl_socket_add_pending_output_no_flush (struct clib_file *uf, - struct vl_api_registration_ *rp, - u8 * buffer, uword buffer_bytes); clib_error_t *vl_socket_write_ready (struct clib_file *uf); void vl_socket_api_send (vl_api_registration_t * rp, u8 * elem); void vl_socket_process_api_msg (clib_file_t * uf, vl_api_registration_t * rp, diff --git a/src/vppinfra/file.h b/src/vppinfra/file.h index b5a0507c3b8..48412ddd99b 100644 --- a/src/vppinfra/file.h +++ b/src/vppinfra/file.h @@ -156,6 +156,13 @@ clib_file_get (clib_file_main_t * fm, u32 file_index) return pool_elt_at_index (fm->file_pool, file_index); } +always_inline clib_error_t * +clib_file_write (clib_file_t * f) +{ + f->write_events++; + return f->write_function (f); +} + #endif /* included_clib_file_h */ /*