X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvcl%2Fldp.c;h=b72c0c54acd4d64afffaf1c0b6af7f929fbb1116;hb=a5a9efd4d;hp=c26c460bec803fcff270fb26c806dcd64541cd03;hpb=54693d23307ce8944a4d97379efd3bd4dcf0485c;p=vpp.git diff --git a/src/vcl/ldp.c b/src/vcl/ldp.c index c26c460bec8..b72c0c54acd 100644 --- a/src/vcl/ldp.c +++ b/src/vcl/ldp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016 Cisco and/or its affiliates. + * Copyright (c) 2016-2019 Cisco and/or its affiliates. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: @@ -26,9 +26,12 @@ #include #include -#include +#include #include #include +#include +#include +#include #define HAVE_CONSTRUCTOR_ATTRIBUTE #ifdef HAVE_CONSTRUCTOR_ATTRIBUTE @@ -46,49 +49,102 @@ #define DESTRUCTOR_ATTRIBUTE #endif -typedef struct +#define LDP_MAX_NWORKERS 32 + +typedef struct ldp_worker_ctx_ { - int init; - char app_name[LDP_APP_NAME_MAX]; - u32 sid_bit_val; - u32 sid_bit_mask; - u32 debug; u8 *io_buffer; clib_time_t clib_time; + + /* + * Select state + */ clib_bitmap_t *rd_bitmap; clib_bitmap_t *wr_bitmap; clib_bitmap_t *ex_bitmap; - clib_bitmap_t *sid_rd_bitmap; - clib_bitmap_t *sid_wr_bitmap; - clib_bitmap_t *sid_ex_bitmap; + clib_bitmap_t *si_rd_bitmap; + clib_bitmap_t *si_wr_bitmap; + clib_bitmap_t *si_ex_bitmap; clib_bitmap_t *libc_rd_bitmap; clib_bitmap_t *libc_wr_bitmap; clib_bitmap_t *libc_ex_bitmap; + + /* + * Poll state + */ vcl_poll_t *vcl_poll; - u8 select_vcl; + struct pollfd *libc_poll; + u16 *libc_poll_idxs; + + /* + * Epoll state + */ u8 epoll_wait_vcl; + u8 mq_epfd_added; + int vcl_mq_epfd; + +} ldp_worker_ctx_t; + +/* clib_bitmap_t, fd_mask and vcl_si_set are used interchangeably. Make sure + * they are the same size */ +STATIC_ASSERT (sizeof (clib_bitmap_t) == sizeof (fd_mask), + "ldp bitmap size mismatch"); +STATIC_ASSERT (sizeof (vcl_si_set) == sizeof (fd_mask), + "ldp bitmap size mismatch"); + +typedef struct +{ + ldp_worker_ctx_t *workers; + int init; + char app_name[LDP_APP_NAME_MAX]; + u32 vlsh_bit_val; + u32 vlsh_bit_mask; + u32 debug; + + /** vcl needs next epoll_create to go to libc_epoll */ + u8 vcl_needs_real_epoll; + + /** + * crypto state used only for testing + */ + u8 transparent_tls; + u32 ckpair_index; } ldp_main_t; + #define LDP_DEBUG ldp->debug +#define LDBG(_lvl, _fmt, _args...) \ + if (ldp->debug > _lvl) \ + { \ + int errno_saved = errno; \ + fprintf (stderr, "ldp<%d>: " _fmt "\n", getpid(), ##_args); \ + errno = errno_saved; \ + } + static ldp_main_t ldp_main = { - .sid_bit_val = (1 << LDP_SID_BIT_MIN), - .sid_bit_mask = (1 << LDP_SID_BIT_MIN) - 1, + .vlsh_bit_val = (1 << LDP_SID_BIT_MIN), + .vlsh_bit_mask = (1 << LDP_SID_BIT_MIN) - 1, .debug = LDP_DEBUG_INIT, + .transparent_tls = 0, + .ckpair_index = ~0, }; static ldp_main_t *ldp = &ldp_main; +static inline ldp_worker_ctx_t * +ldp_worker_get_current (void) +{ + return (ldp->workers + vppcom_worker_index ()); +} + /* * RETURN: 0 on success or -1 on error. * */ static inline void ldp_set_app_name (char *app_name) { - int rv = snprintf (ldp->app_name, LDP_APP_NAME_MAX, - "ldp-%d-%s", getpid (), app_name); - - if (rv >= LDP_APP_NAME_MAX) - app_name[LDP_APP_NAME_MAX - 1] = 0; + snprintf (ldp->app_name, LDP_APP_NAME_MAX, + "ldp-%d-%s", getpid (), app_name); } static inline char * @@ -101,157 +157,160 @@ ldp_get_app_name () } static inline int -ldp_fd_from_sid (u32 sid) +ldp_vlsh_to_fd (vls_handle_t vlsh) { - if (PREDICT_FALSE (sid >= ldp->sid_bit_val)) - return -EMFILE; - else - return (sid | ldp->sid_bit_val); + return (vlsh + ldp->vlsh_bit_val); } -static inline int -ldp_fd_is_sid (int fd) +static inline vls_handle_t +ldp_fd_to_vlsh (int fd) { - return ((u32) fd & ldp->sid_bit_val) ? 1 : 0; + if (fd < ldp->vlsh_bit_val) + return VLS_INVALID_HANDLE; + + return (fd - ldp->vlsh_bit_val); } -static inline u32 -ldp_sid_from_fd (int fd) +static void +ldp_alloc_workers (void) { - return (ldp_fd_is_sid (fd) ? ((u32) fd & ldp->sid_bit_mask) : - INVALID_SESSION_ID); + if (ldp->workers) + return; + pool_alloc (ldp->workers, LDP_MAX_NWORKERS); } static inline int ldp_init (void) { - int rv = 0; + ldp_worker_ctx_t *ldpw; + int rv; + + if (PREDICT_TRUE (ldp->init)) + return 0; - if (PREDICT_FALSE (!ldp->init)) + ldp->init = 1; + ldp->vcl_needs_real_epoll = 1; + rv = vls_app_create (ldp_get_app_name ()); + if (rv != VPPCOM_OK) { - ldp->init = 1; - rv = vppcom_app_create (ldp_get_app_name ()); - if (rv == VPPCOM_OK) + ldp->vcl_needs_real_epoll = 0; + if (rv == VPPCOM_EEXIST) + return 0; + LDBG (2, "\nERROR: ldp_init: vppcom_app_create()" + " failed! rv = %d (%s)\n", rv, vppcom_retval_str (rv)); + ldp->init = 0; + return rv; + } + ldp->vcl_needs_real_epoll = 0; + ldp_alloc_workers (); + ldpw = ldp_worker_get_current (); + + char *env_var_str = getenv (LDP_ENV_DEBUG); + if (env_var_str) + { + u32 tmp; + if (sscanf (env_var_str, "%u", &tmp) != 1) + clib_warning ("LDP<%d>: WARNING: Invalid LDP debug level specified in" + " the env var " LDP_ENV_DEBUG " (%s)!", getpid (), + env_var_str); + else { - char *env_var_str = getenv (LDP_ENV_DEBUG); - if (env_var_str) - { - u32 tmp; - if (sscanf (env_var_str, "%u", &tmp) != 1) - clib_warning ("LDP<%d>: WARNING: Invalid LDP debug level " - "specified in the env var " LDP_ENV_DEBUG - " (%s)!", getpid (), env_var_str); - else - { - ldp->debug = tmp; - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: configured LDP debug level (%u) " - "from the env var " LDP_ENV_DEBUG "!", - getpid (), ldp->debug); - } - } + ldp->debug = tmp; + LDBG (0, "configured LDP debug level (%u) from env var " + LDP_ENV_DEBUG "!", ldp->debug); + } + } - env_var_str = getenv (LDP_ENV_APP_NAME); - if (env_var_str) - { - ldp_set_app_name (env_var_str); - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: configured LDP app name (%s) " - "from the env var " LDP_ENV_APP_NAME "!", - getpid (), ldp->app_name); - } + env_var_str = getenv (LDP_ENV_APP_NAME); + if (env_var_str) + { + ldp_set_app_name (env_var_str); + LDBG (0, "configured LDP app name (%s) from the env var " + LDP_ENV_APP_NAME "!", ldp->app_name); + } - env_var_str = getenv (LDP_ENV_SID_BIT); - if (env_var_str) - { - u32 sb; - if (sscanf (env_var_str, "%u", &sb) != 1) - { - clib_warning ("LDP<%d>: WARNING: Invalid LDP sid bit " - "specified in the env var " - LDP_ENV_SID_BIT " (%s)!" - "sid bit value %d (0x%x)", - getpid (), env_var_str, - ldp->sid_bit_val, ldp->sid_bit_val); - } - else if (sb < LDP_SID_BIT_MIN) - { - ldp->sid_bit_val = (1 << LDP_SID_BIT_MIN); - ldp->sid_bit_mask = ldp->sid_bit_val - 1; - - clib_warning ("LDP<%d>: WARNING: LDP sid bit (%u) " - "specified in the env var " - LDP_ENV_SID_BIT " (%s) is too small. " - "Using LDP_SID_BIT_MIN (%d)! " - "sid bit value %d (0x%x)", - getpid (), sb, env_var_str, LDP_SID_BIT_MIN, - ldp->sid_bit_val, ldp->sid_bit_val); - } - else if (sb > LDP_SID_BIT_MAX) - { - ldp->sid_bit_val = (1 << LDP_SID_BIT_MAX); - ldp->sid_bit_mask = ldp->sid_bit_val - 1; - - clib_warning ("LDP<%d>: WARNING: LDP sid bit (%u) " - "specified in the env var " - LDP_ENV_SID_BIT " (%s) is too big. " - "Using LDP_SID_BIT_MAX (%d)! " - "sid bit value %d (0x%x)", - getpid (), sb, env_var_str, LDP_SID_BIT_MAX, - ldp->sid_bit_val, ldp->sid_bit_val); - } - else - { - ldp->sid_bit_val = (1 << sb); - ldp->sid_bit_mask = ldp->sid_bit_val - 1; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: configured LDP sid bit (%u) " - "from " LDP_ENV_SID_BIT - "! sid bit value %d (0x%x)", getpid (), - sb, ldp->sid_bit_val, ldp->sid_bit_val); - } - } + env_var_str = getenv (LDP_ENV_SID_BIT); + if (env_var_str) + { + u32 sb; + if (sscanf (env_var_str, "%u", &sb) != 1) + { + LDBG (0, "WARNING: Invalid LDP sid bit specified in the env var " + LDP_ENV_SID_BIT " (%s)! sid bit value %d (0x%x)", env_var_str, + ldp->vlsh_bit_val, ldp->vlsh_bit_val); + } + else if (sb < LDP_SID_BIT_MIN) + { + ldp->vlsh_bit_val = (1 << LDP_SID_BIT_MIN); + ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1; + + LDBG (0, "WARNING: LDP sid bit (%u) specified in the env var " + LDP_ENV_SID_BIT " (%s) is too small. Using LDP_SID_BIT_MIN" + " (%d)! sid bit value %d (0x%x)", sb, env_var_str, + LDP_SID_BIT_MIN, ldp->vlsh_bit_val, ldp->vlsh_bit_val); + } + else if (sb > LDP_SID_BIT_MAX) + { + ldp->vlsh_bit_val = (1 << LDP_SID_BIT_MAX); + ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1; - clib_time_init (&ldp->clib_time); - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: LDP initialization: done!", getpid ()); + LDBG (0, "WARNING: LDP sid bit (%u) specified in the env var " + LDP_ENV_SID_BIT " (%s) is too big. Using LDP_SID_BIT_MAX" + " (%d)! sid bit value %d (0x%x)", sb, env_var_str, + LDP_SID_BIT_MAX, ldp->vlsh_bit_val, ldp->vlsh_bit_val); } else { - fprintf (stderr, "\nLDP<%d>: ERROR: ldp_init: vppcom_app_create()" - " failed! rv = %d (%s)\n", - getpid (), rv, vppcom_retval_str (rv)); + ldp->vlsh_bit_val = (1 << sb); + ldp->vlsh_bit_mask = ldp->vlsh_bit_val - 1; + + LDBG (0, "configured LDP sid bit (%u) from " + LDP_ENV_SID_BIT "! sid bit value %d (0x%x)", sb, + ldp->vlsh_bit_val, ldp->vlsh_bit_val); + } + + /* Make sure there are enough bits in the fd set for vcl sessions */ + if (ldp->vlsh_bit_val > FD_SETSIZE / 2) + { + LDBG (0, "ERROR: LDP vlsh bit value %d > FD_SETSIZE/2 %d!", + ldp->vlsh_bit_val, FD_SETSIZE / 2); ldp->init = 0; + return -1; } } - return rv; + env_var_str = getenv (LDP_ENV_TLS_TRANS); + if (env_var_str) + { + ldp->transparent_tls = 1; + } + + /* *INDENT-OFF* */ + pool_foreach (ldpw, ldp->workers) { + clib_memset (&ldpw->clib_time, 0, sizeof (ldpw->clib_time)); + } + /* *INDENT-ON* */ + + LDBG (0, "LDP initialization: done!"); + + return 0; } int close (int fd) { - int rv; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); + vls_handle_t vlsh; + int rv, epfd; if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - int epfd; - - func_str = "vppcom_session_attr[GET_LIBC_EPFD]"; - epfd = vppcom_session_attr (sid, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); + epfd = vls_attr (vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); if (epfd > 0) { - func_str = "libc_close"; - - if (LDP_DEBUG > 0) - clib_warning - ("LDP<%d>: fd %d (0x%x): calling %s(): epfd %u (0x%x)", - getpid (), fd, fd, func_str, epfd, epfd); + LDBG (0, "fd %d: calling libc_close: epfd %u", fd, epfd); rv = libc_close (epfd); if (rv < 0) @@ -259,8 +318,7 @@ close (int fd) u32 size = sizeof (epfd); epfd = 0; - (void) vppcom_session_attr (sid, VPPCOM_ATTR_SET_LIBC_EPFD, - &epfd, &size); + (void) vls_attr (vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &epfd, &size); } } else if (PREDICT_FALSE (epfd < 0)) @@ -270,13 +328,9 @@ close (int fd) goto done; } - func_str = "vppcom_session_close"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); + LDBG (0, "fd %d: calling vls_close: vlsh %u", fd, vlsh); - rv = vppcom_session_close (sid); + rv = vls_close (vlsh); if (rv != VPPCOM_OK) { errno = -rv; @@ -285,54 +339,27 @@ close (int fd) } else { - func_str = "libc_close"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s()", - getpid (), fd, fd, func_str); - + LDBG (0, "fd %d: calling libc_close", fd); rv = libc_close (fd); } done: - if (LDP_DEBUG > 0) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } ssize_t read (int fd, void *buf, size_t nbytes) { + vls_handle_t vlsh; ssize_t size; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "vppcom_session_read"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), buf %p, nbytes %u", getpid (), - fd, fd, func_str, sid, sid, buf, nbytes); - - size = vppcom_session_read (sid, buf, nbytes); + size = vls_read (vlsh, buf, nbytes); if (size < 0) { errno = -size; @@ -341,79 +368,38 @@ read (int fd, void *buf, size_t nbytes) } else { - func_str = "libc_read"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "buf %p, nbytes %u", getpid (), - fd, fd, func_str, buf, nbytes); - size = libc_read (fd, buf, nbytes); } - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); - } return size; } ssize_t readv (int fd, const struct iovec * iov, int iovcnt) { - const char *func_str; - ssize_t size = 0; - u32 sid = ldp_sid_from_fd (fd); int rv = 0, i, total = 0; + vls_handle_t vlsh; + ssize_t size = 0; if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "vppcom_session_read"; - do + for (i = 0; i < iovcnt; ++i) { - for (i = 0; i < iovcnt; ++i) + rv = vls_read (vlsh, iov[i].iov_base, iov[i].iov_len); + if (rv <= 0) + break; + else { - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s() [%d]: " - "sid %u (0x%x), iov %p, iovcnt %d, total %d", - getpid (), fd, fd, func_str, i, sid, sid, - iov, iovcnt, total); - - rv = vppcom_session_read (sid, iov[i].iov_base, iov[i].iov_len); - if (rv < 0) + total += rv; + if (rv < iov[i].iov_len) break; - else - { - total += rv; - if (rv < iov[i].iov_len) - { - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): " - "rv (%d) < iov[%d].iov_len (%d)", - getpid (), fd, fd, rv, i, - iov[i].iov_len); - break; - } - } } } - while ((rv >= 0) && (total == 0)); - - if (rv < 0) + if (rv < 0 && total == 0) { errno = -rv; size = -1; @@ -423,53 +409,25 @@ readv (int fd, const struct iovec * iov, int iovcnt) } else { - func_str = "libc_readv"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "iov %p, iovcnt %d", getpid (), fd, fd, iov, iovcnt); - size = libc_readv (fd, iov, iovcnt); } - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); - } return size; } ssize_t write (int fd, const void *buf, size_t nbytes) { - const char *func_str; + vls_handle_t vlsh; ssize_t size = 0; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "vppcom_session_write"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), buf %p, nbytes %u", getpid (), - fd, fd, func_str, sid, sid, buf, nbytes); - - size = vppcom_session_write (sid, (void *) buf, nbytes); + size = vls_write_msg (vlsh, (void *) buf, nbytes); if (size < 0) { errno = -size; @@ -478,84 +436,39 @@ write (int fd, const void *buf, size_t nbytes) } else { - func_str = "libc_write"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "buf %p, nbytes %u", getpid (), - fd, fd, func_str, buf, nbytes); - size = libc_write (fd, buf, nbytes); } - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); - } return size; } ssize_t writev (int fd, const struct iovec * iov, int iovcnt) { - const char *func_str; ssize_t size = 0, total = 0; - u32 sid = ldp_sid_from_fd (fd); + vls_handle_t vlsh; int i, rv = 0; - /* - * Use [f]printf() instead of clib_warning() to prevent recursion SIGSEGV. - */ - if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "vppcom_session_write"; - do + for (i = 0; i < iovcnt; ++i) { - for (i = 0; i < iovcnt; ++i) + rv = vls_write_msg (vlsh, iov[i].iov_base, iov[i].iov_len); + if (rv < 0) + break; + else { - if (LDP_DEBUG > 4) - printf ("%s:%d: LDP<%d>: fd %d (0x%x): calling %s() [%d]: " - "sid %u (0x%x), buf %p, nbytes %ld, total %ld", - __func__, __LINE__, getpid (), fd, fd, func_str, - i, sid, sid, iov[i].iov_base, iov[i].iov_len, total); - - rv = vppcom_session_write (sid, iov[i].iov_base, - iov[i].iov_len); - if (rv < 0) + total += rv; + if (rv < iov[i].iov_len) break; - else - { - total += rv; - if (rv < iov[i].iov_len) - { - if (LDP_DEBUG > 4) - printf ("%s:%d: LDP<%d>: fd %d (0x%x): " - "rv (%d) < iov[%d].iov_len (%ld)", - __func__, __LINE__, getpid (), fd, fd, - rv, i, iov[i].iov_len); - break; - } - } } } - while ((rv >= 0) && (total == 0)); - if (rv < 0) + if (rv < 0 && total == 0) { errno = -rv; size = -1; @@ -565,48 +478,21 @@ writev (int fd, const struct iovec * iov, int iovcnt) } else { - func_str = "libc_writev"; - - if (LDP_DEBUG > 4) - printf ("%s:%d: LDP<%d>: fd %d (0x%x): calling %s(): " - "iov %p, iovcnt %d\n", __func__, __LINE__, getpid (), - fd, fd, func_str, iov, iovcnt); - size = libc_writev (fd, iov, iovcnt); } - if (LDP_DEBUG > 4) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - fprintf (stderr, - "%s:%d: LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %ld, errno = %d\n", __func__, __LINE__, getpid (), fd, - fd, func_str, size, errno_val); - errno = errno_val; - } - else - printf ("%s:%d: LDP<%d>: fd %d (0x%x): returning %ld\n", - __func__, __LINE__, getpid (), fd, fd, size); - } return size; } -int -fcntl (int fd, int cmd, ...) +static int +fcntl_internal (int fd, int cmd, va_list ap) { - const char *func_str = __func__; + vls_handle_t vlsh; int rv = 0; - va_list ap; - u32 sid = ldp_sid_from_fd (fd); - - if ((errno = -ldp_init ())) - return -1; - va_start (ap, cmd); - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + LDBG (0, "fd %u vlsh %d, cmd %u", fd, vlsh, cmd); + if (vlsh != VLS_INVALID_HANDLE) { int flags = va_arg (ap, int); u32 size; @@ -616,37 +502,19 @@ fcntl (int fd, int cmd, ...) switch (cmd) { case F_SETFL: - func_str = "vppcom_session_attr[SET_FLAGS]"; - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x) flags %d (0x%x), size %d", - getpid (), fd, fd, func_str, sid, sid, - flags, flags, size); - - rv = - vppcom_session_attr (sid, VPPCOM_ATTR_SET_FLAGS, &flags, &size); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_FLAGS, &flags, &size); break; case F_GETFL: - func_str = "vppcom_session_attr[GET_FLAGS]"; - if (LDP_DEBUG > 2) - clib_warning - ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), " - "flags %d (0x%x), size %d", getpid (), fd, fd, func_str, sid, - sid, flags, flags, size); - - rv = - vppcom_session_attr (sid, VPPCOM_ATTR_GET_FLAGS, &flags, &size); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_FLAGS, &flags, &size); if (rv == VPPCOM_OK) - { - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x), cmd %d (F_GETFL): " - "%s() returned flags %d (0x%x)", - getpid (), fd, fd, cmd, func_str, flags, flags); - rv = flags; - } + rv = flags; + break; + case F_SETFD: + /* TODO handle this */ + LDBG (0, "F_SETFD ignored flags %u", flags); + rv = 0; break; - default: rv = -EOPNOTSUPP; break; @@ -659,60 +527,66 @@ fcntl (int fd, int cmd, ...) } else { - func_str = "libc_vfcntl"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): cmd %d", - getpid (), fd, fd, func_str, cmd); - +#ifdef HAVE_FCNTL64 + rv = libc_vfcntl64 (fd, cmd, ap); +#else rv = libc_vfcntl (fd, cmd, ap); +#endif } + return rv; +} + +int +fcntl (int fd, int cmd, ...) +{ + va_list ap; + int rv; + + if ((errno = -ldp_init ())) + return -1; + + va_start (ap, cmd); + rv = fcntl_internal (fd, cmd, ap); va_end (ap); - if (LDP_DEBUG > 2) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } int -ioctl (int fd, unsigned long int cmd, ...) +fcntl64 (int fd, int cmd, ...) { - const char *func_str; + va_list ap; int rv; + + if ((errno = -ldp_init ())) + return -1; + + va_start (ap, cmd); + rv = fcntl_internal (fd, cmd, ap); + va_end (ap); + return rv; +} + +int +ioctl (int fd, unsigned long int cmd, ...) +{ + vls_handle_t vlsh; va_list ap; - u32 sid = ldp_sid_from_fd (fd); + int rv; if ((errno = -ldp_init ())) return -1; va_start (ap, cmd); - if (sid != INVALID_SESSION_ID) - { - func_str = "vppcom_session_attr[GET_NREAD]"; + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) + { switch (cmd) { case FIONREAD: - if (LDP_DEBUG > 2) - clib_warning - ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_NREAD, 0, 0); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_NREAD, 0, 0); break; case FIONBIO: @@ -724,14 +598,7 @@ ioctl (int fd, unsigned long int cmd, ...) * non-blocking, the flags should be read here and merged * with O_NONBLOCK. */ - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), flags %d (0x%x), size %d", - getpid (), fd, fd, func_str, sid, sid, - flags, flags, size); - - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_FLAGS, &flags, - &size); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_FLAGS, &flags, &size); } break; @@ -745,36 +612,92 @@ ioctl (int fd, unsigned long int cmd, ...) rv = -1; } } - else - { - func_str = "libc_vioctl"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): cmd %d", - getpid (), fd, fd, func_str, cmd); - - rv = libc_vioctl (fd, cmd, ap); - } - - if (LDP_DEBUG > 2) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } + else + { + rv = libc_vioctl (fd, cmd, ap); + } + va_end (ap); return rv; } +always_inline void +ldp_select_init_maps (fd_set * __restrict original, + clib_bitmap_t ** resultb, clib_bitmap_t ** libcb, + clib_bitmap_t ** vclb, int nfds, u32 minbits, + u32 n_bytes, uword * si_bits, uword * libc_bits) +{ + uword si_bits_set, libc_bits_set; + vls_handle_t vlsh; + int fd; + + clib_bitmap_validate (*vclb, minbits); + clib_bitmap_validate (*libcb, minbits); + clib_bitmap_validate (*resultb, minbits); + clib_memcpy_fast (*resultb, original, n_bytes); + memset (original, 0, n_bytes); + + /* *INDENT-OFF* */ + clib_bitmap_foreach (fd, *resultb) { + if (fd > nfds) + break; + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh == VLS_INVALID_HANDLE) + clib_bitmap_set_no_check (*libcb, fd, 1); + else + *vclb = clib_bitmap_set (*vclb, vlsh_to_session_index (vlsh), 1); + } + /* *INDENT-ON* */ + + si_bits_set = clib_bitmap_last_set (*vclb) + 1; + *si_bits = (si_bits_set > *si_bits) ? si_bits_set : *si_bits; + clib_bitmap_validate (*resultb, *si_bits); + + libc_bits_set = clib_bitmap_last_set (*libcb) + 1; + *libc_bits = (libc_bits_set > *libc_bits) ? libc_bits_set : *libc_bits; +} + +always_inline int +ldp_select_vcl_map_to_libc (clib_bitmap_t * vclb, fd_set * __restrict libcb) +{ + vls_handle_t vlsh; + uword si; + int fd; + + if (!libcb) + return 0; + + /* *INDENT-OFF* */ + clib_bitmap_foreach (si, vclb) { + vlsh = vls_session_index_to_vlsh (si); + ASSERT (vlsh != VLS_INVALID_HANDLE); + fd = ldp_vlsh_to_fd (vlsh); + if (PREDICT_FALSE (fd < 0)) + { + errno = EBADFD; + return -1; + } + FD_SET (fd, libcb); + } + /* *INDENT-ON* */ + + return 0; +} + +always_inline void +ldp_select_libc_map_merge (clib_bitmap_t * result, fd_set * __restrict libcb) +{ + uword fd; + + if (!libcb) + return; + + /* *INDENT-OFF* */ + clib_bitmap_foreach (fd, result) + FD_SET ((int)fd, libcb); + /* *INDENT-ON* */ +} + int ldp_pselect (int nfds, fd_set * __restrict readfds, fd_set * __restrict writefds, @@ -782,13 +705,12 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, const struct timespec *__restrict timeout, const __sigset_t * __restrict sigmask) { - int rv; - char *func_str = "##"; - f64 time_out; - int fd; - uword sid_bits, sid_bits_set, libc_bits, libc_bits_set; - u32 minbits = clib_max (nfds, BITS (uword)); - u32 sid; + u32 minbits = clib_max (nfds, BITS (uword)), n_bytes; + ldp_worker_ctx_t *ldpw = ldp_worker_get_current (); + struct timespec libc_tspec = { 0 }; + f64 time_out, vcl_timeout = 0; + uword si_bits, libc_bits; + int rv, bits_set = 0; if (nfds < 0) { @@ -796,21 +718,19 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, return -1; } + if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0)) + clib_time_init (&ldpw->clib_time); + if (timeout) { time_out = (timeout->tv_sec == 0 && timeout->tv_nsec == 0) ? - (f64) 0 : (f64) timeout->tv_sec + - (f64) timeout->tv_nsec / (f64) 1000000000; + (f64) 0 : (f64) timeout->tv_sec + (f64) timeout->tv_nsec / (f64) 1e9; /* select as fine grained sleep */ if (!nfds) { - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: sleeping for %.02f seconds", - getpid (), time_out); - - time_out += clib_time_now (&ldp->clib_time); - while (clib_time_now (&ldp->clib_time) < time_out) + time_out += clib_time_now (&ldpw->clib_time); + while (clib_time_now (&ldpw->clib_time) < time_out) ; return 0; } @@ -823,283 +743,137 @@ ldp_pselect (int nfds, fd_set * __restrict readfds, else time_out = -1; - - if (nfds <= ldp->sid_bit_val) + if (nfds <= ldp->vlsh_bit_val) { - func_str = "libc_pselect"; - - if (LDP_DEBUG > 3) - clib_warning - ("LDP<%d>: calling %s(): nfds %d, readfds %p, writefds %p, " - "exceptfds %p, timeout %p, sigmask %p", getpid (), func_str, nfds, - readfds, writefds, exceptfds, timeout, sigmask); - rv = libc_pselect (nfds, readfds, writefds, exceptfds, timeout, sigmask); goto done; } - if (PREDICT_FALSE (ldp->sid_bit_val > FD_SETSIZE / 2)) - { - clib_warning ("LDP<%d>: ERROR: LDP sid bit value %d (0x%x) > " - "FD_SETSIZE/2 %d (0x%x)!", getpid (), - ldp->sid_bit_val, ldp->sid_bit_val, - FD_SETSIZE / 2, FD_SETSIZE / 2); - errno = EOVERFLOW; - return -1; - } + si_bits = libc_bits = 0; + n_bytes = nfds / 8 + ((nfds % 8) ? 1 : 0); - sid_bits = libc_bits = 0; if (readfds) - { - clib_bitmap_validate (ldp->sid_rd_bitmap, minbits); - clib_bitmap_validate (ldp->libc_rd_bitmap, minbits); - clib_bitmap_validate (ldp->rd_bitmap, minbits); - clib_memcpy (ldp->rd_bitmap, readfds, - vec_len (ldp->rd_bitmap) * sizeof (clib_bitmap_t)); - FD_ZERO (readfds); - - /* *INDENT-OFF* */ - clib_bitmap_foreach (fd, ldp->rd_bitmap, - ({ - sid = ldp_sid_from_fd (fd); - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: readfds: fd %d (0x%x), sid %u (0x%x)", - getpid (), fd, fd, sid, sid); - if (sid == INVALID_SESSION_ID) - clib_bitmap_set_no_check (ldp->libc_rd_bitmap, fd, 1); - else - clib_bitmap_set_no_check (ldp->sid_rd_bitmap, sid, 1); - })); - /* *INDENT-ON* */ - - sid_bits_set = clib_bitmap_last_set (ldp->sid_rd_bitmap) + 1; - sid_bits = (sid_bits_set > sid_bits) ? sid_bits_set : sid_bits; - - libc_bits_set = clib_bitmap_last_set (ldp->libc_rd_bitmap) + 1; - libc_bits = (libc_bits_set > libc_bits) ? libc_bits_set : libc_bits; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: readfds: sid_bits_set %d, sid_bits %d, " - "libc_bits_set %d, libc_bits %d", getpid (), - sid_bits_set, sid_bits, libc_bits_set, libc_bits); - } + ldp_select_init_maps (readfds, &ldpw->rd_bitmap, &ldpw->libc_rd_bitmap, + &ldpw->si_rd_bitmap, nfds, minbits, n_bytes, + &si_bits, &libc_bits); if (writefds) - { - clib_bitmap_validate (ldp->sid_wr_bitmap, minbits); - clib_bitmap_validate (ldp->libc_wr_bitmap, minbits); - clib_bitmap_validate (ldp->wr_bitmap, minbits); - clib_memcpy (ldp->wr_bitmap, writefds, - vec_len (ldp->wr_bitmap) * sizeof (clib_bitmap_t)); - FD_ZERO (writefds); - - /* *INDENT-OFF* */ - clib_bitmap_foreach (fd, ldp->wr_bitmap, - ({ - sid = ldp_sid_from_fd (fd); - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: writefds: fd %d (0x%x), sid %u (0x%x)", - getpid (), fd, fd, sid, sid); - if (sid == INVALID_SESSION_ID) - clib_bitmap_set_no_check (ldp->libc_wr_bitmap, fd, 1); - else - clib_bitmap_set_no_check (ldp->sid_wr_bitmap, sid, 1); - })); - /* *INDENT-ON* */ - - sid_bits_set = clib_bitmap_last_set (ldp->sid_wr_bitmap) + 1; - sid_bits = (sid_bits_set > sid_bits) ? sid_bits_set : sid_bits; - - libc_bits_set = clib_bitmap_last_set (ldp->libc_wr_bitmap) + 1; - libc_bits = (libc_bits_set > libc_bits) ? libc_bits_set : libc_bits; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: writefds: sid_bits_set %d, sid_bits %d, " - "libc_bits_set %d, libc_bits %d", getpid (), - sid_bits_set, sid_bits, libc_bits_set, libc_bits); - } + ldp_select_init_maps (writefds, &ldpw->wr_bitmap, + &ldpw->libc_wr_bitmap, &ldpw->si_wr_bitmap, nfds, + minbits, n_bytes, &si_bits, &libc_bits); if (exceptfds) - { - clib_bitmap_validate (ldp->sid_ex_bitmap, minbits); - clib_bitmap_validate (ldp->libc_ex_bitmap, minbits); - clib_bitmap_validate (ldp->ex_bitmap, minbits); - clib_memcpy (ldp->ex_bitmap, exceptfds, - vec_len (ldp->ex_bitmap) * sizeof (clib_bitmap_t)); - FD_ZERO (exceptfds); - - /* *INDENT-OFF* */ - clib_bitmap_foreach (fd, ldp->ex_bitmap, - ({ - sid = ldp_sid_from_fd (fd); - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: exceptfds: fd %d (0x%x), sid %u (0x%x)", - getpid (), fd, fd, sid, sid); - if (sid == INVALID_SESSION_ID) - clib_bitmap_set_no_check (ldp->libc_ex_bitmap, fd, 1); - else - clib_bitmap_set_no_check (ldp->sid_ex_bitmap, sid, 1); - })); - /* *INDENT-ON* */ - - sid_bits_set = clib_bitmap_last_set (ldp->sid_ex_bitmap) + 1; - sid_bits = (sid_bits_set > sid_bits) ? sid_bits_set : sid_bits; - - libc_bits_set = clib_bitmap_last_set (ldp->libc_ex_bitmap) + 1; - libc_bits = (libc_bits_set > libc_bits) ? libc_bits_set : libc_bits; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: exceptfds: sid_bits_set %d, sid_bits %d, " - "libc_bits_set %d, libc_bits %d", getpid (), - sid_bits_set, sid_bits, libc_bits_set, libc_bits); - } - - if (PREDICT_FALSE (!sid_bits && !libc_bits)) + ldp_select_init_maps (exceptfds, &ldpw->ex_bitmap, + &ldpw->libc_ex_bitmap, &ldpw->si_ex_bitmap, nfds, + minbits, n_bytes, &si_bits, &libc_bits); + + if (PREDICT_FALSE (!si_bits && !libc_bits)) { errno = EINVAL; rv = -1; goto done; } + if (!si_bits) + libc_tspec = timeout ? *timeout : libc_tspec; + do { - if (sid_bits) + if (si_bits) { - if (!ldp->select_vcl) + if (readfds) + clib_memcpy_fast (ldpw->rd_bitmap, ldpw->si_rd_bitmap, + vec_len (ldpw->si_rd_bitmap) * + sizeof (clib_bitmap_t)); + if (writefds) + clib_memcpy_fast (ldpw->wr_bitmap, ldpw->si_wr_bitmap, + vec_len (ldpw->si_wr_bitmap) * + sizeof (clib_bitmap_t)); + if (exceptfds) + clib_memcpy_fast (ldpw->ex_bitmap, ldpw->si_ex_bitmap, + vec_len (ldpw->si_ex_bitmap) * + sizeof (clib_bitmap_t)); + + rv = vls_select (si_bits, readfds ? ldpw->rd_bitmap : NULL, + writefds ? ldpw->wr_bitmap : NULL, + exceptfds ? ldpw->ex_bitmap : NULL, vcl_timeout); + if (rv < 0) + { + errno = -rv; + rv = -1; + goto done; + } + else if (rv > 0) { - func_str = "vppcom_select"; - - if (readfds) - clib_memcpy (ldp->rd_bitmap, ldp->sid_rd_bitmap, - vec_len (ldp->rd_bitmap) * - sizeof (clib_bitmap_t)); - if (writefds) - clib_memcpy (ldp->wr_bitmap, ldp->sid_wr_bitmap, - vec_len (ldp->wr_bitmap) * - sizeof (clib_bitmap_t)); - if (exceptfds) - clib_memcpy (ldp->ex_bitmap, ldp->sid_ex_bitmap, - vec_len (ldp->ex_bitmap) * - sizeof (clib_bitmap_t)); - - rv = vppcom_select (sid_bits, - readfds ? ldp->rd_bitmap : NULL, - writefds ? ldp->wr_bitmap : NULL, - exceptfds ? ldp->ex_bitmap : NULL, 0); - if (rv < 0) + if (ldp_select_vcl_map_to_libc (ldpw->rd_bitmap, readfds)) { - errno = -rv; rv = -1; + goto done; } - else if (rv > 0) + + if (ldp_select_vcl_map_to_libc (ldpw->wr_bitmap, writefds)) { - if (readfds) - { - /* *INDENT-OFF* */ - clib_bitmap_foreach (sid, ldp->rd_bitmap, - ({ - fd = ldp_fd_from_sid (sid); - if (PREDICT_FALSE (fd < 0)) - { - errno = EBADFD; - rv = -1; - goto done; - } - FD_SET (fd, readfds); - })); - /* *INDENT-ON* */ - } - if (writefds) - { - /* *INDENT-OFF* */ - clib_bitmap_foreach (sid, ldp->wr_bitmap, - ({ - fd = ldp_fd_from_sid (sid); - if (PREDICT_FALSE (fd < 0)) - { - errno = EBADFD; - rv = -1; - goto done; - } - FD_SET (fd, writefds); - })); - /* *INDENT-ON* */ - } - if (exceptfds) - { - /* *INDENT-OFF* */ - clib_bitmap_foreach (sid, ldp->ex_bitmap, - ({ - fd = ldp_fd_from_sid (sid); - if (PREDICT_FALSE (fd < 0)) - { - errno = EBADFD; - rv = -1; - goto done; - } - FD_SET (fd, exceptfds); - })); - /* *INDENT-ON* */ - } - ldp->select_vcl = 1; + rv = -1; + goto done; + } + + if (ldp_select_vcl_map_to_libc (ldpw->ex_bitmap, exceptfds)) + { + rv = -1; goto done; } + bits_set = rv; } - else - ldp->select_vcl = 0; } if (libc_bits) { - struct timespec tspec; - - func_str = "libc_pselect"; - if (readfds) - clib_memcpy (readfds, ldp->libc_rd_bitmap, - vec_len (ldp->rd_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldpw->rd_bitmap, ldpw->libc_rd_bitmap, + vec_len (ldpw->libc_rd_bitmap) * + sizeof (clib_bitmap_t)); if (writefds) - clib_memcpy (writefds, ldp->libc_wr_bitmap, - vec_len (ldp->wr_bitmap) * sizeof (clib_bitmap_t)); + clib_memcpy_fast (ldpw->wr_bitmap, ldpw->libc_wr_bitmap, + vec_len (ldpw->libc_wr_bitmap) * + sizeof (clib_bitmap_t)); if (exceptfds) - clib_memcpy (exceptfds, ldp->libc_ex_bitmap, - vec_len (ldp->ex_bitmap) * sizeof (clib_bitmap_t)); - tspec.tv_sec = tspec.tv_nsec = 0; + clib_memcpy_fast (ldpw->ex_bitmap, ldpw->libc_ex_bitmap, + vec_len (ldpw->libc_ex_bitmap) * + sizeof (clib_bitmap_t)); + rv = libc_pselect (libc_bits, - readfds ? readfds : NULL, - writefds ? writefds : NULL, - exceptfds ? exceptfds : NULL, &tspec, sigmask); - if (rv != 0) - goto done; + readfds ? (fd_set *) ldpw->rd_bitmap : NULL, + writefds ? (fd_set *) ldpw->wr_bitmap : NULL, + exceptfds ? (fd_set *) ldpw->ex_bitmap : NULL, + &libc_tspec, sigmask); + if (rv > 0) + { + ldp_select_libc_map_merge (ldpw->rd_bitmap, readfds); + ldp_select_libc_map_merge (ldpw->wr_bitmap, writefds); + ldp_select_libc_map_merge (ldpw->ex_bitmap, exceptfds); + bits_set += rv; + } + } + + if (bits_set) + { + rv = bits_set; + goto done; } } - while ((time_out == -1) || (clib_time_now (&ldp->clib_time) < time_out)); + while ((time_out == -1) || (clib_time_now (&ldpw->clib_time) < time_out)); rv = 0; done: /* TBD: set timeout to amount of time left */ - vec_reset_length (ldp->rd_bitmap); - vec_reset_length (ldp->sid_rd_bitmap); - vec_reset_length (ldp->libc_rd_bitmap); - vec_reset_length (ldp->wr_bitmap); - vec_reset_length (ldp->sid_wr_bitmap); - vec_reset_length (ldp->libc_wr_bitmap); - vec_reset_length (ldp->ex_bitmap); - vec_reset_length (ldp->sid_ex_bitmap); - vec_reset_length (ldp->libc_ex_bitmap); - - if (LDP_DEBUG > 3) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: %s() failed! " - "rv %d, errno = %d", getpid (), - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: returning %d (0x%x)", getpid (), rv, rv); - } + clib_bitmap_zero (ldpw->rd_bitmap); + clib_bitmap_zero (ldpw->si_rd_bitmap); + clib_bitmap_zero (ldpw->libc_rd_bitmap); + clib_bitmap_zero (ldpw->wr_bitmap); + clib_bitmap_zero (ldpw->si_wr_bitmap); + clib_bitmap_zero (ldpw->libc_wr_bitmap); + clib_bitmap_zero (ldpw->ex_bitmap); + clib_bitmap_zero (ldpw->si_ex_bitmap); + clib_bitmap_zero (ldpw->libc_ex_bitmap); + return rv; } @@ -1131,13 +905,80 @@ pselect (int nfds, fd_set * __restrict readfds, } #endif +/* If transparent TLS mode is turned on, then ldp will load key and cert. + */ +static int +load_cert_key_pair (void) +{ + char *cert_str = getenv (LDP_ENV_TLS_CERT); + char *key_str = getenv (LDP_ENV_TLS_KEY); + char cert_buf[4096], key_buf[4096]; + int cert_size, key_size; + vppcom_cert_key_pair_t crypto; + int ckp_index; + FILE *fp; + + if (!cert_str || !key_str) + { + LDBG (0, "ERROR: failed to read LDP environment %s\n", + LDP_ENV_TLS_CERT); + return -1; + } + + fp = fopen (cert_str, "r"); + if (fp == NULL) + { + LDBG (0, "ERROR: failed to open cert file %s \n", cert_str); + return -1; + } + cert_size = fread (cert_buf, sizeof (char), sizeof (cert_buf), fp); + fclose (fp); + + fp = fopen (key_str, "r"); + if (fp == NULL) + { + LDBG (0, "ERROR: failed to open key file %s \n", key_str); + return -1; + } + key_size = fread (key_buf, sizeof (char), sizeof (key_buf), fp); + fclose (fp); + + crypto.cert = cert_buf; + crypto.key = key_buf; + crypto.cert_len = cert_size; + crypto.key_len = key_size; + ckp_index = vppcom_add_cert_key_pair (&crypto); + if (ckp_index < 0) + { + LDBG (0, "ERROR: failed to add cert key pair\n"); + return -1; + } + + ldp->ckpair_index = ckp_index; + + return 0; +} + +static int +assign_cert_key_pair (vls_handle_t vlsh) +{ + uint32_t ckp_len; + + if (ldp->ckpair_index == ~0 && load_cert_key_pair () < 0) + return -1; + + ckp_len = sizeof (ldp->ckpair_index); + return vppcom_session_attr (vlsh_to_session_index (vlsh), + VPPCOM_ATTR_SET_CKPAIR, &ldp->ckpair_index, + &ckp_len); +} + int socket (int domain, int type, int protocol) { - const char *func_str; - int rv; + int rv, sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK); u8 is_nonblocking = type & SOCK_NONBLOCK ? 1 : 0; - int sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK); + vls_handle_t vlsh; if ((errno = -ldp_init ())) return -1; @@ -1145,60 +986,40 @@ socket (int domain, int type, int protocol) if (((domain == AF_INET) || (domain == AF_INET6)) && ((sock_type == SOCK_STREAM) || (sock_type == SOCK_DGRAM))) { - int sid; - u8 proto = ((sock_type == SOCK_DGRAM) ? - VPPCOM_PROTO_UDP : VPPCOM_PROTO_TCP); - - func_str = "vppcom_session_create"; + u8 proto; + if (ldp->transparent_tls) + { + proto = VPPCOM_PROTO_TLS; + } + else + proto = ((sock_type == SOCK_DGRAM) ? + VPPCOM_PROTO_UDP : VPPCOM_PROTO_TCP); - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: : calling %s(): " - "proto %u (%s), is_nonblocking %u", - getpid (), func_str, proto, - vppcom_proto_str (proto), is_nonblocking); + LDBG (0, "calling vls_create: proto %u (%s), is_nonblocking %u", + proto, vppcom_proto_str (proto), is_nonblocking); - sid = vppcom_session_create (proto, is_nonblocking); - if (sid < 0) + vlsh = vls_create (proto, is_nonblocking); + if (vlsh < 0) { - errno = -sid; + errno = -vlsh; rv = -1; } else { - func_str = "ldp_fd_from_sid"; - rv = ldp_fd_from_sid (sid); - if (rv < 0) + if (ldp->transparent_tls) { - (void) vppcom_session_close (sid); - errno = -rv; - rv = -1; + if (assign_cert_key_pair (vlsh) < 0) + return -1; } + rv = ldp_vlsh_to_fd (vlsh); } } else { - func_str = "libc_socket"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: : calling %s()", getpid (), func_str); - + LDBG (0, "calling libc_socket"); rv = libc_socket (domain, type, protocol); } - if (LDP_DEBUG > 0) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: %s() failed! " - "rv %d, errno = %d", - getpid (), func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: : returning fd %d (0x%x)", getpid (), rv, rv); - } return rv; } @@ -1212,9 +1033,7 @@ socket (int domain, int type, int protocol) int socketpair (int domain, int type, int protocol, int fds[2]) { - const char *func_str; - int rv; - int sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK); + int rv, sock_type = type & ~(SOCK_CLOEXEC | SOCK_NONBLOCK); if ((errno = -ldp_init ())) return -1; @@ -1222,63 +1041,40 @@ socketpair (int domain, int type, int protocol, int fds[2]) if (((domain == AF_INET) || (domain == AF_INET6)) && ((sock_type == SOCK_STREAM) || (sock_type == SOCK_DGRAM))) { - func_str = __func__; - - clib_warning ("LDP<%d>: LDP-TBD", getpid ()); + LDBG (0, "LDP-TBD"); errno = ENOSYS; rv = -1; } else { - func_str = "libc_socket"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: : calling %s()", getpid (), func_str); - - rv = libc_socket (domain, type, protocol); + LDBG (1, "calling libc_socketpair"); + rv = libc_socketpair (domain, type, protocol, fds); } - if (LDP_DEBUG > 1) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: %s() failed! " - "rv %d, errno = %d", - getpid (), func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: : returning fd %d (0x%x)", getpid (), rv, rv); - } return rv; } int bind (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) { + vls_handle_t vlsh; int rv; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { vppcom_endpt_t ep; - func_str = "vppcom_session_bind"; - switch (addr->sa_family) { case AF_INET: if (len != sizeof (struct sockaddr_in)) { - clib_warning - ("LDP<%d>: ERROR: fd %d (0x%x): sid %u (0x%x): Invalid " - "AF_INET addr len %u!", getpid (), fd, fd, sid, sid, len); + LDBG (0, "ERROR: fd %d: vlsh %u: Invalid AF_INET addr len %u!", + fd, vlsh, len); errno = EINVAL; rv = -1; goto done; @@ -1291,9 +1087,8 @@ bind (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) case AF_INET6: if (len != sizeof (struct sockaddr_in6)) { - clib_warning - ("LDP<%d>: ERROR: fd %d (0x%x): sid %u (0x%x): Invalid " - "AF_INET6 addr len %u!", getpid (), fd, fd, sid, sid, len); + LDBG (0, "ERROR: fd %d: vlsh %u: Invalid AF_INET6 addr len %u!", + fd, vlsh, len); errno = EINVAL; rv = -1; goto done; @@ -1304,19 +1099,16 @@ bind (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) break; default: - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): sid %u (0x%x): " - "Unsupported address family %u!", - getpid (), fd, fd, sid, sid, addr->sa_family); + LDBG (0, "ERROR: fd %d: vlsh %u: Unsupported address family %u!", + fd, vlsh, addr->sa_family); errno = EAFNOSUPPORT; rv = -1; goto done; } - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), " - "addr %p, len %u", - getpid (), fd, fd, func_str, sid, sid, addr, len); + LDBG (0, "fd %d: calling vls_bind: vlsh %u, addr %p, len %u", fd, vlsh, + addr, len); - rv = vppcom_session_bind (sid, &ep); + rv = vls_bind (vlsh, &ep); if (rv != VPPCOM_OK) { errno = -rv; @@ -1325,32 +1117,13 @@ bind (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) } else { - func_str = "libc_bind"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "addr %p, len %u", - getpid (), fd, fd, func_str, addr, len); - + LDBG (0, "fd %d: calling libc_bind: addr %p, len %u", fd, addr, len); rv = libc_bind (fd, addr, len); } done: - if (LDP_DEBUG > 0) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } + LDBG (1, "fd %d: returning %d", fd, rv); + return rv; } @@ -1403,28 +1176,22 @@ ldp_copy_ep_to_sockaddr (__SOCKADDR_ARG addr, socklen_t * __restrict len, int getsockname (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len) { + vls_handle_t vlsh; int rv; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { vppcom_endpt_t ep; u8 addr_buf[sizeof (struct in6_addr)]; u32 size = sizeof (ep); ep.ip = addr_buf; - func_str = "vppcom_session_attr[GET_LCL_ADDR]"; - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), " - "addr %p, len %u", - getpid (), fd, fd, func_str, sid, sid, addr, len); - - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_LCL_ADDR, &ep, &size); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_LCL_ADDR, &ep, &size); if (rv != VPPCOM_OK) { errno = -rv; @@ -1442,67 +1209,41 @@ getsockname (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len) } else { - func_str = "libc_getsockname"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "addr %p, len %u", - getpid (), fd, fd, func_str, addr, len); - rv = libc_getsockname (fd, addr, len); } - if (LDP_DEBUG > 2) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } int connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) { + vls_handle_t vlsh; int rv; - const char *func_str = __func__; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; if (!addr) { - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): NULL addr, len %u", - getpid (), fd, fd, len); + LDBG (0, "ERROR: fd %d: NULL addr, len %u", fd, len); errno = EINVAL; rv = -1; goto done; } - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { vppcom_endpt_t ep; - func_str = "vppcom_session_connect"; - switch (addr->sa_family) { case AF_INET: if (len != sizeof (struct sockaddr_in)) { - clib_warning - ("LDP<%d>: fd %d (0x%x): ERROR sid %u (0x%x): Invalid " - "AF_INET addr len %u!", getpid (), fd, fd, sid, sid, len); + LDBG (0, "fd %d: ERROR vlsh %u: Invalid AF_INET addr len %u!", + fd, vlsh, len); errno = EINVAL; rv = -1; goto done; @@ -1515,9 +1256,8 @@ connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) case AF_INET6: if (len != sizeof (struct sockaddr_in6)) { - clib_warning - ("LDP<%d>: fd %d (0x%x): ERROR sid %u (0x%x): Invalid " - "AF_INET6 addr len %u!", getpid (), fd, fd, sid, sid, len); + LDBG (0, "fd %d: ERROR vlsh %u: Invalid AF_INET6 addr len %u!", + fd, vlsh, len); errno = EINVAL; rv = -1; goto done; @@ -1528,19 +1268,16 @@ connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) break; default: - clib_warning ("LDP<%d>: fd %d (0x%x): ERROR sid %u (0x%x): " - "Unsupported address family %u!", - getpid (), fd, fd, sid, sid, addr->sa_family); + LDBG (0, "fd %d: ERROR vlsh %u: Unsupported address family %u!", + fd, vlsh, addr->sa_family); errno = EAFNOSUPPORT; rv = -1; goto done; } - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x) " - "addr %p len %u", - getpid (), fd, fd, func_str, sid, sid, addr, len); + LDBG (0, "fd %d: calling vls_connect(): vlsh %u addr %p len %u", fd, + vlsh, addr, len); - rv = vppcom_session_connect (sid, &ep); + rv = vls_connect (vlsh, &ep); if (rv != VPPCOM_OK) { errno = -rv; @@ -1549,60 +1286,35 @@ connect (int fd, __CONST_SOCKADDR_ARG addr, socklen_t len) } else { - func_str = "libc_connect"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "addr %p, len %u", - getpid (), fd, fd, func_str, addr, len); + LDBG (0, "fd %d: calling libc_connect(): addr %p, len %u", + fd, addr, len); rv = libc_connect (fd, addr, len); } done: - if (LDP_DEBUG > 0) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } + LDBG (1, "fd %d: returning %d (0x%x)", fd, rv, rv); return rv; } int getpeername (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len) { + vls_handle_t vlsh; int rv; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { vppcom_endpt_t ep; u8 addr_buf[sizeof (struct in6_addr)]; u32 size = sizeof (ep); ep.ip = addr_buf; - func_str = "vppcom_session_attr[GET_PEER_ADDR]"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), " - "addr %p, len %u", - getpid (), fd, fd, func_str, sid, sid, addr, len); - - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_PEER_ADDR, &ep, &size); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_PEER_ADDR, &ep, &size); if (rv != VPPCOM_OK) { errno = -rv; @@ -1620,123 +1332,66 @@ getpeername (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict len) } else { - func_str = "libc_getpeername"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "addr %p, len %u", - getpid (), fd, fd, func_str, addr, len); - rv = libc_getpeername (fd, addr, len); } - if (LDP_DEBUG > 2) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } ssize_t send (int fd, const void *buf, size_t n, int flags) { + vls_handle_t vlsh = ldp_fd_to_vlsh (fd); ssize_t size; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + if (vlsh != VLS_INVALID_HANDLE) { - - func_str = "vppcom_session_sendto"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), " - "buf %p, n %u, flags 0x%x", - getpid (), fd, fd, func_str, sid, sid, buf, n, flags); - - size = vppcom_session_sendto (sid, (void *) buf, n, flags, NULL); + size = vls_sendto (vlsh, (void *) buf, n, flags, NULL); if (size < VPPCOM_OK) { - errno = -size; - size = -1; - } - } - else - { - func_str = "libc_send"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "buf %p, n %u, flags 0x%x", - getpid (), fd, fd, func_str, buf, n, flags); - - size = libc_send (fd, buf, n, flags); - } - - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; + errno = -size; + size = -1; } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); } + else + { + size = libc_send (fd, buf, n, flags); + } + return size; } ssize_t sendfile (int out_fd, int in_fd, off_t * offset, size_t len) { + ldp_worker_ctx_t *ldpw = ldp_worker_get_current (); + vls_handle_t vlsh; ssize_t size = 0; - const char *func_str; - u32 sid = ldp_sid_from_fd (out_fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (out_fd); + if (vlsh != VLS_INVALID_HANDLE) { int rv; ssize_t results = 0; size_t n_bytes_left = len; size_t bytes_to_read; int nbytes; - int errno_val; u8 eagain = 0; u32 flags, flags_len = sizeof (flags); - func_str = "vppcom_session_attr[GET_FLAGS]"; - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_FLAGS, &flags, - &flags_len); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_FLAGS, &flags, &flags_len); if (PREDICT_FALSE (rv != VPPCOM_OK)) { - clib_warning ("LDP<%d>: ERROR: out fd %d (0x%x): %s(): " - "sid %u (0x%x), returned %d (%s)!", getpid (), - out_fd, out_fd, func_str, sid, sid, rv, - vppcom_retval_str (rv)); + LDBG (0, "ERROR: out fd %d: vls_attr: vlsh %u, returned %d (%s)!", + out_fd, vlsh, rv, vppcom_retval_str (rv)); - vec_reset_length (ldp->io_buffer); + vec_reset_length (ldpw->io_buffer); errno = -rv; size = -1; goto done; @@ -1747,14 +1402,6 @@ sendfile (int out_fd, int in_fd, off_t * offset, size_t len) off_t off = lseek (in_fd, *offset, SEEK_SET); if (PREDICT_FALSE (off == -1)) { - func_str = "lseek"; - errno_val = errno; - clib_warning ("LDP<%d>: ERROR: out fd %d (0x%x): %s(): " - "SEEK_SET failed: in_fd %d, offset %p, " - "*offset %ld, rv %ld, errno %d", getpid (), - out_fd, out_fd, in_fd, offset, *offset, off, - errno_val); - errno = errno_val; size = -1; goto done; } @@ -1764,76 +1411,44 @@ sendfile (int out_fd, int in_fd, off_t * offset, size_t len) do { - func_str = "vppcom_session_attr[GET_NWRITE]"; - size = vppcom_session_attr (sid, VPPCOM_ATTR_GET_NWRITE, 0, 0); + size = vls_attr (vlsh, VPPCOM_ATTR_GET_NWRITE, 0, 0); if (size < 0) { - clib_warning - ("LDP<%d>: ERROR: fd %d (0x%x): %s(): sid %u (0x%x), " - "returned %d (%s)!", getpid (), out_fd, out_fd, func_str, - sid, sid, size, vppcom_retval_str (size)); - vec_reset_length (ldp->io_buffer); + LDBG (0, "ERROR: fd %d: vls_attr: vlsh %u returned %ld (%s)!", + out_fd, vlsh, size, vppcom_retval_str (size)); + vec_reset_length (ldpw->io_buffer); errno = -size; size = -1; goto done; } bytes_to_read = size; - if (LDP_DEBUG > 2) - clib_warning - ("LDP<%d>: fd %d (0x%x): called %s(): sid %u (0x%x), " - "results %ld, n_bytes_left %lu, bytes_to_read %lu", getpid (), - out_fd, out_fd, func_str, sid, sid, results, n_bytes_left, - bytes_to_read); - if (bytes_to_read == 0) { if (flags & O_NONBLOCK) { if (!results) - { - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): sid %u (0x%x): " - "EAGAIN", - getpid (), out_fd, out_fd, sid, sid); - eagain = 1; - } + eagain = 1; goto update_offset; } else continue; } bytes_to_read = clib_min (n_bytes_left, bytes_to_read); - vec_validate (ldp->io_buffer, bytes_to_read); - nbytes = libc_read (in_fd, ldp->io_buffer, bytes_to_read); + vec_validate (ldpw->io_buffer, bytes_to_read); + nbytes = libc_read (in_fd, ldpw->io_buffer, bytes_to_read); if (nbytes < 0) { - func_str = "libc_read"; - errno_val = errno; - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): in_fd (%d), " - "io_buffer %p, bytes_to_read %lu, rv %d, " - "errno %d", getpid (), out_fd, out_fd, func_str, - in_fd, ldp->io_buffer, bytes_to_read, nbytes, - errno_val); - errno = errno_val; - if (results == 0) { - vec_reset_length (ldp->io_buffer); + vec_reset_length (ldpw->io_buffer); size = -1; goto done; } goto update_offset; } - func_str = "vppcom_session_write"; - if (LDP_DEBUG > 2) - clib_warning - ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), " - "buf %p, nbytes %u: results %d, n_bytes_left %d", getpid (), - out_fd, out_fd, func_str, sid, sid, ldp->io_buffer, nbytes, - results, n_bytes_left); - - size = vppcom_session_write (sid, ldp->io_buffer, nbytes); + + size = vls_write (vlsh, ldpw->io_buffer, nbytes); if (size < 0) { if (size == VPPCOM_EAGAIN) @@ -1841,30 +1456,15 @@ sendfile (int out_fd, int in_fd, off_t * offset, size_t len) if (flags & O_NONBLOCK) { if (!results) - { - if (LDP_DEBUG > 2) - clib_warning - ("LDP<%d>: fd %d (0x%x): sid %u (0x%x): " - "EAGAIN", getpid (), out_fd, out_fd, sid, sid); - eagain = 1; - } + eagain = 1; goto update_offset; } else continue; } - else - { - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s():" - "sid %u, io_buffer %p, nbytes %u " - "returned %d (%s)", - getpid (), out_fd, out_fd, func_str, - sid, ldp->io_buffer, nbytes, - size, vppcom_retval_str (size)); - } if (results == 0) { - vec_reset_length (ldp->io_buffer); + vec_reset_length (ldpw->io_buffer); errno = -size; size = -1; goto done; @@ -1879,19 +1479,12 @@ sendfile (int out_fd, int in_fd, off_t * offset, size_t len) while (n_bytes_left > 0); update_offset: - vec_reset_length (ldp->io_buffer); + vec_reset_length (ldpw->io_buffer); if (offset) { off_t off = lseek (in_fd, *offset, SEEK_SET); if (PREDICT_FALSE (off == -1)) { - func_str = "lseek"; - errno_val = errno; - clib_warning ("LDP<%d>: ERROR: %s(): SEEK_SET failed: " - "in_fd %d, offset %p, *offset %ld, " - "rv %ld, errno %d", getpid (), in_fd, - offset, *offset, off, errno_val); - errno = errno_val; size = -1; goto done; } @@ -1909,33 +1502,10 @@ sendfile (int out_fd, int in_fd, off_t * offset, size_t len) } else { - func_str = "libc_send"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "in_fd %d, offset %p, len %u", - getpid (), out_fd, out_fd, func_str, - in_fd, offset, len); - size = libc_sendfile (out_fd, in_fd, offset, len); } done: - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), out_fd, out_fd, - func_str, size, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), out_fd, out_fd, size, size); - } return size; } @@ -1948,23 +1518,16 @@ sendfile64 (int out_fd, int in_fd, off_t * offset, size_t len) ssize_t recv (int fd, void *buf, size_t n, int flags) { + vls_handle_t vlsh; ssize_t size; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "vppcom_session_recvfrom"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), buf %p, n %u, flags 0x%x", getpid (), - fd, fd, func_str, sid, sid, buf, n, flags); - - size = vppcom_session_recvfrom (sid, buf, n, flags, NULL); + size = vls_recvfrom (vlsh, buf, n, flags, NULL); if (size < 0) { errno = -size; @@ -1973,31 +1536,72 @@ recv (int fd, void *buf, size_t n, int flags) } else { - func_str = "libc_recv"; + size = libc_recv (fd, buf, n, flags); + } - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "buf %p, n %u, flags 0x%x", getpid (), - fd, fd, func_str, buf, n, flags); + return size; +} - size = libc_recv (fd, buf, n, flags); +static int +ldp_vls_sendo (vls_handle_t vlsh, const void *buf, size_t n, int flags, + __CONST_SOCKADDR_ARG addr, socklen_t addr_len) +{ + vppcom_endpt_t *ep = 0; + vppcom_endpt_t _ep; + + if (addr) + { + ep = &_ep; + switch (addr->sa_family) + { + case AF_INET: + ep->is_ip4 = VPPCOM_IS_IP4; + ep->ip = + (uint8_t *) & ((const struct sockaddr_in *) addr)->sin_addr; + ep->port = (uint16_t) ((const struct sockaddr_in *) addr)->sin_port; + break; + + case AF_INET6: + ep->is_ip4 = VPPCOM_IS_IP6; + ep->ip = + (uint8_t *) & ((const struct sockaddr_in6 *) addr)->sin6_addr; + ep->port = + (uint16_t) ((const struct sockaddr_in6 *) addr)->sin6_port; + break; + + default: + return EAFNOSUPPORT; + } } - if (LDP_DEBUG > 2) + return vls_sendto (vlsh, (void *) buf, n, flags, ep); +} + +static int +ldp_vls_recvfrom (vls_handle_t vlsh, void *__restrict buf, size_t n, + int flags, __SOCKADDR_ARG addr, + socklen_t * __restrict addr_len) +{ + u8 src_addr[sizeof (struct sockaddr_in6)]; + vppcom_endpt_t ep; + ssize_t size; + int rv; + + if (addr) { - if (size < 0) + ep.ip = src_addr; + size = vls_recvfrom (vlsh, buf, n, flags, &ep); + + if (size > 0) { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; + rv = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep); + if (rv < 0) + size = rv; } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); } + else + size = vls_recvfrom (vlsh, buf, n, flags, NULL); + return size; } @@ -2005,55 +1609,16 @@ ssize_t sendto (int fd, const void *buf, size_t n, int flags, __CONST_SOCKADDR_ARG addr, socklen_t addr_len) { + vls_handle_t vlsh; ssize_t size; - const char *func_str = __func__; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != INVALID_SESSION_ID) { - vppcom_endpt_t *ep = 0; - vppcom_endpt_t _ep; - - if (addr) - { - ep = &_ep; - switch (addr->sa_family) - { - case AF_INET: - ep->is_ip4 = VPPCOM_IS_IP4; - ep->ip = - (uint8_t *) & ((const struct sockaddr_in *) addr)->sin_addr; - ep->port = - (uint16_t) ((const struct sockaddr_in *) addr)->sin_port; - break; - - case AF_INET6: - ep->is_ip4 = VPPCOM_IS_IP6; - ep->ip = - (uint8_t *) & ((const struct sockaddr_in6 *) addr)->sin6_addr; - ep->port = - (uint16_t) ((const struct sockaddr_in6 *) addr)->sin6_port; - break; - - default: - errno = EAFNOSUPPORT; - size = -1; - goto done; - } - } - - func_str = "vppcom_session_sendto"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), buf %p, n %u, flags 0x%x, ep %p", - getpid (), fd, fd, func_str, sid, sid, buf, n, - flags, ep); - - size = vppcom_session_sendto (sid, (void *) buf, n, flags, ep); + size = ldp_vls_sendo (vlsh, buf, n, flags, addr, addr_len); if (size < 0) { errno = -size; @@ -2062,33 +1627,9 @@ sendto (int fd, const void *buf, size_t n, int flags, } else { - func_str = "libc_sendto"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "buf %p, n %u, flags 0x%x, addr %p, addr_len %d", - getpid (), fd, fd, func_str, buf, n, flags, - addr, addr_len); - size = libc_sendto (fd, buf, n, flags, addr, addr_len); } -done: - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); - } return size; } @@ -2096,36 +1637,16 @@ ssize_t recvfrom (int fd, void *__restrict buf, size_t n, int flags, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len) { + vls_handle_t vlsh; ssize_t size; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - vppcom_endpt_t ep; - u8 src_addr[sizeof (struct sockaddr_in6)]; - - func_str = "vppcom_session_recvfrom"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), buf %p, n %u, flags 0x%x, ep %p", - getpid (), fd, fd, func_str, sid, sid, buf, n, - flags, &ep); - if (addr) - { - ep.ip = src_addr; - size = vppcom_session_recvfrom (sid, buf, n, flags, &ep); - - if (size > 0) - size = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep); - } - else - size = vppcom_session_recvfrom (sid, buf, n, flags, NULL); - + size = ldp_vls_recvfrom (vlsh, buf, n, flags, addr, addr_len); if (size < 0) { errno = -size; @@ -2134,80 +1655,55 @@ recvfrom (int fd, void *__restrict buf, size_t n, int flags, } else { - func_str = "libc_recvfrom"; - - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "buf %p, n %u, flags 0x%x, addr %p, addr_len %d", - getpid (), fd, fd, func_str, buf, n, flags, - addr, addr_len); - size = libc_recvfrom (fd, buf, n, flags, addr, addr_len); } - if (LDP_DEBUG > 2) - { - if (size < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); - } return size; } ssize_t -sendmsg (int fd, const struct msghdr * message, int flags) +sendmsg (int fd, const struct msghdr * msg, int flags) { + vls_handle_t vlsh; ssize_t size; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) - { - func_str = __func__; - - clib_warning ("LDP<%d>: LDP-TBD", getpid ()); - errno = ENOSYS; - size = -1; - } - else + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "libc_sendmsg"; + struct iovec *iov = msg->msg_iov; + ssize_t total = 0; + int i, rv; - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "message %p, flags 0x%x", - getpid (), fd, fd, func_str, message, flags); - - size = libc_sendmsg (fd, message, flags); - } + for (i = 0; i < msg->msg_iovlen; ++i) + { + rv = ldp_vls_sendo (vlsh, iov[i].iov_base, iov[i].iov_len, flags, + msg->msg_name, msg->msg_namelen); + if (rv < 0) + break; + else + { + total += rv; + if (rv < iov[i].iov_len) + break; + } + } - if (LDP_DEBUG > 2) - { - if (size < 0) + if (rv < 0 && total == 0) { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; + errno = -rv; + size = -1; } else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); + size = total; + } + else + { + size = libc_sendmsg (fd, msg, flags); } + return size; } @@ -2217,12 +1713,12 @@ sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags) { ssize_t size; const char *func_str; - u32 sid = ldp_sid_from_fd (fd); + u32 sh = ldp_fd_to_vlsh (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + if (sh != INVALID_SESSION_ID) { clib_warning ("LDP<%d>: LDP-TBD", getpid ()); errno = ENOSYS; @@ -2260,50 +1756,55 @@ sendmmsg (int fd, struct mmsghdr *vmessages, unsigned int vlen, int flags) #endif ssize_t -recvmsg (int fd, struct msghdr * message, int flags) +recvmsg (int fd, struct msghdr * msg, int flags) { + vls_handle_t vlsh; ssize_t size; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) - { - func_str = __func__; - - clib_warning ("LDP<%d>: LDP-TBD", getpid ()); - errno = ENOSYS; - size = -1; - } - else + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "libc_recvmsg"; + struct iovec *iov = msg->msg_iov; + ssize_t max_deq, total = 0; + int i, rv; - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "message %p, flags 0x%x", - getpid (), fd, fd, func_str, message, flags); + max_deq = vls_attr (vlsh, VPPCOM_ATTR_GET_NREAD, 0, 0); + if (!max_deq) + return 0; - size = libc_recvmsg (fd, message, flags); - } + for (i = 0; i < msg->msg_iovlen; i++) + { + rv = ldp_vls_recvfrom (vlsh, iov[i].iov_base, iov[i].iov_len, flags, + (i == 0 ? msg->msg_name : NULL), + (i == 0 ? &msg->msg_namelen : NULL)); + if (rv <= 0) + break; + else + { + total += rv; + if (rv < iov[i].iov_len) + break; + } + if (total >= max_deq) + break; + } - if (LDP_DEBUG > 2) - { - if (size < 0) + if (rv < 0 && total == 0) { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, size, errno_val); - errno = errno_val; + errno = -rv; + size = -1; } else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, size, size); + size = total; } + else + { + size = libc_recvmsg (fd, msg, flags); + } + return size; } @@ -2314,12 +1815,12 @@ recvmmsg (int fd, struct mmsghdr *vmessages, { ssize_t size; const char *func_str; - u32 sid = ldp_sid_from_fd (fd); + u32 sh = ldp_fd_to_vlsh (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + if (sh != INVALID_SESSION_ID) { clib_warning ("LDP<%d>: LDP-TBD", getpid ()); errno = ENOSYS; @@ -2361,15 +1862,14 @@ int getsockopt (int fd, int level, int optname, void *__restrict optval, socklen_t * __restrict optlen) { + vls_handle_t vlsh; int rv; - const char *func_str = __func__; - u32 sid = ldp_sid_from_fd (fd); - u32 buflen = optlen ? (u32) * optlen : 0; if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { rv = -EOPNOTSUPP; @@ -2379,62 +1879,40 @@ getsockopt (int fd, int level, int optname, switch (optname) { case TCP_NODELAY: - func_str = "vppcom_session_attr[SOL_TCP,GET_TCP_NODELAY]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_TCP_NODELAY, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_NODELAY, + optval, optlen); break; case TCP_MAXSEG: - func_str = "vppcom_session_attr[SOL_TCP,GET_TCP_USER_MSS]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_TCP_USER_MSS, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_USER_MSS, + optval, optlen); break; case TCP_KEEPIDLE: - func_str = "vppcom_session_attr[SOL_TCP,GET_TCP_KEEPIDLE]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_TCP_KEEPIDLE, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_KEEPIDLE, + optval, optlen); break; case TCP_KEEPINTVL: - func_str = "vppcom_session_attr[SOL_TCP,GET_TCP_KEEPINTVL]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), SOL_TCP", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_TCP_KEEPINTVL, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TCP_KEEPINTVL, + optval, optlen); break; case TCP_INFO: if (optval && optlen && (*optlen == sizeof (struct tcp_info))) { - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): sid %u (0x%x), " - "SOL_TCP, TCP_INFO, optval %p, " - "optlen %d: #LDP-NOP#", - getpid (), fd, fd, sid, sid, - optval, *optlen); + LDBG (1, "fd %d: vlsh %u SOL_TCP, TCP_INFO, optval %p, " + "optlen %d: #LDP-NOP#", fd, vlsh, optval, *optlen); memset (optval, 0, *optlen); rv = VPPCOM_OK; } else rv = -EFAULT; break; + case TCP_CONGESTION: + *optlen = strlen ("cubic"); + strncpy (optval, "cubic", *optlen + 1); + rv = 0; + break; default: - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): " - "sid %u (0x%x), SOL_TCP, " - "optname %d unsupported!", - getpid (), fd, fd, func_str, sid, sid, optname); + LDBG (0, "ERROR: fd %d: getsockopt SOL_TCP: sid %u, " + "optname %d unsupported!", fd, vlsh, optname); break; } break; @@ -2442,20 +1920,11 @@ getsockopt (int fd, int level, int optname, switch (optname) { case IPV6_V6ONLY: - func_str = "vppcom_session_attr[SOL_IPV6,GET_V6ONLY]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_V6ONLY, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_V6ONLY, optval, optlen); break; default: - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): " - "sid %u (0x%x), SOL_IPV6, " - "optname %d unsupported!", - getpid (), fd, fd, func_str, sid, sid, optname); + LDBG (0, "ERROR: fd %d: getsockopt SOL_IPV6: vlsh %u " + "optname %d unsupported!", fd, vlsh, optname); break; } break; @@ -2463,84 +1932,35 @@ getsockopt (int fd, int level, int optname, switch (optname) { case SO_ACCEPTCONN: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_ACCEPTCONN]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_LISTEN, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_LISTEN, optval, optlen); break; case SO_KEEPALIVE: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_KEEPALIVE]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_KEEPALIVE, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_KEEPALIVE, optval, optlen); break; case SO_PROTOCOL: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_PROTOCOL]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_PROTOCOL, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_PROTOCOL, optval, optlen); *(int *) optval = *(int *) optval ? SOCK_DGRAM : SOCK_STREAM; break; case SO_SNDBUF: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_TX_FIFO_LEN]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), optlen %d", - getpid (), fd, fd, func_str, sid, sid, buflen); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_TX_FIFO_LEN, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_TX_FIFO_LEN, + optval, optlen); break; case SO_RCVBUF: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_RX_FIFO_LEN]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), optlen %d", - getpid (), fd, fd, func_str, sid, sid, buflen); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_RX_FIFO_LEN, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_RX_FIFO_LEN, + optval, optlen); break; case SO_REUSEADDR: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_REUSEADDR]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_REUSEADDR, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_REUSEADDR, optval, optlen); break; case SO_BROADCAST: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_BROADCAST]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_BROADCAST, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_BROADCAST, optval, optlen); break; case SO_ERROR: - func_str = "vppcom_session_attr[SOL_SOCKET,GET_ERROR]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_GET_ERROR, - optval, optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_GET_ERROR, optval, optlen); break; default: - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): " - "sid %u (0x%x), SOL_SOCKET, " - "optname %d unsupported!", - getpid (), fd, fd, func_str, sid, sid, optname); + LDBG (0, "ERROR: fd %d: getsockopt SOL_SOCKET: vlsh %u " + "optname %d unsupported!", fd, vlsh, optname); break; } break; @@ -2556,32 +1976,9 @@ getsockopt (int fd, int level, int optname, } else { - func_str = "libc_getsockopt"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): level %d, " - "optname %d, optval %p, optlen %d", - getpid (), fd, fd, func_str, level, optname, - optval, optlen); - rv = libc_getsockopt (fd, level, optname, optval, optlen); } - if (LDP_DEBUG > 1) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } @@ -2589,14 +1986,14 @@ int setsockopt (int fd, int level, int optname, const void *optval, socklen_t optlen) { + vls_handle_t vlsh; int rv; - const char *func_str = __func__; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { rv = -EOPNOTSUPP; @@ -2606,47 +2003,29 @@ setsockopt (int fd, int level, int optname, switch (optname) { case TCP_NODELAY: - func_str = "vppcom_session_attr[SOL_TCP,SET_TCP_NODELAY]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_TCP_NODELAY, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_NODELAY, + (void *) optval, &optlen); break; case TCP_MAXSEG: - func_str = "vppcom_session_attr[SOL_TCP,SET_TCP_USER_MSS]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_TCP_USER_MSS, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_USER_MSS, + (void *) optval, &optlen); break; case TCP_KEEPIDLE: - func_str = "vppcom_session_attr[SOL_TCP,SET_TCP_KEEPIDLE]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_TCP_KEEPIDLE, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_KEEPIDLE, + (void *) optval, &optlen); break; case TCP_KEEPINTVL: - func_str = "vppcom_session_attr[SOL_TCP,SET_TCP_KEEPINTVL]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x), SOL_TCP", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_TCP_KEEPINTVL, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_TCP_KEEPINTVL, + (void *) optval, &optlen); + break; + case TCP_CONGESTION: + case TCP_CORK: + /* Ignore */ + rv = 0; break; default: - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): " - "sid %u (0x%x), SOL_TCP, " - "optname %d unsupported!", - getpid (), fd, fd, func_str, sid, sid, optname); + LDBG (0, "ERROR: fd %d: setsockopt() SOL_TCP: vlsh %u" + "optname %d unsupported!", fd, vlsh, optname); break; } break; @@ -2654,20 +2033,12 @@ setsockopt (int fd, int level, int optname, switch (optname) { case IPV6_V6ONLY: - func_str = "vppcom_session_attr[SOL_IPV6,SET_V6ONLY]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_V6ONLY, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_V6ONLY, + (void *) optval, &optlen); break; default: - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): " - "sid %u (0x%x), SOL_IPV6, " - "optname %d unsupported!", - getpid (), fd, fd, func_str, sid, sid, optname); + LDBG (0, "ERROR: fd %d: setsockopt SOL_IPV6: vlsh %u" + "optname %d unsupported!", fd, vlsh, optname); break; } break; @@ -2675,38 +2046,20 @@ setsockopt (int fd, int level, int optname, switch (optname) { case SO_KEEPALIVE: - func_str = "vppcom_session_attr[SOL_SOCKET,SET_KEEPALIVE]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_KEEPALIVE, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_KEEPALIVE, + (void *) optval, &optlen); break; case SO_REUSEADDR: - func_str = "vppcom_session_attr[SOL_SOCKET,SET_REUSEADDR]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_REUSEADDR, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_REUSEADDR, + (void *) optval, &optlen); break; case SO_BROADCAST: - func_str = "vppcom_session_attr[SOL_SOCKET,SET_BROADCAST]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): " - "sid %u (0x%x)", - getpid (), fd, fd, func_str, sid, sid); - rv = vppcom_session_attr (sid, VPPCOM_ATTR_SET_BROADCAST, - (void *) optval, &optlen); + rv = vls_attr (vlsh, VPPCOM_ATTR_SET_BROADCAST, + (void *) optval, &optlen); break; default: - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s(): " - "sid %u (0x%x), SOL_SOCKET, " - "optname %d unsupported!", - getpid (), fd, fd, func_str, sid, sid, optname); + LDBG (0, "ERROR: fd %d: setsockopt SOL_SOCKET: vlsh %u " + "optname %d unsupported!", fd, vlsh, optname); break; } break; @@ -2722,55 +2075,27 @@ setsockopt (int fd, int level, int optname, } else { - func_str = "libc_setsockopt"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): level %d, " - "optname %d, optval %p, optlen %d", - getpid (), fd, fd, func_str, level, optname, - optval, optlen); - - rv = libc_setsockopt (fd, level, optname, optval, optlen); - } - - if (LDP_DEBUG > 1) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); + rv = libc_setsockopt (fd, level, optname, optval, optlen); } + return rv; } int listen (int fd, int n) { + vls_handle_t vlsh; int rv; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = "vppcom_session_listen"; - - if (LDP_DEBUG > 0) - clib_warning - ("LDP<%d>: fd %d (0x%x): calling %s(): sid %u (0x%x), n %d", - getpid (), fd, fd, func_str, sid, sid, n); + LDBG (0, "fd %d: calling vls_listen: vlsh %u, n %d", fd, vlsh, n); - rv = vppcom_session_listen (sid, n); + rv = vls_listen (vlsh, n); if (rv != VPPCOM_OK) { errno = -rv; @@ -2779,30 +2104,11 @@ listen (int fd, int n) } else { - func_str = "libc_listen"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): n %d", - getpid (), fd, fd, func_str, n); - + LDBG (0, "fd %d: calling libc_listen(): n %d", fd, n); rv = libc_listen (fd, n); } - if (LDP_DEBUG > 0) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } + LDBG (1, "fd %d: returning %d", fd, rv); return rv; } @@ -2810,33 +2116,27 @@ static inline int ldp_accept4 (int listen_fd, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len, int flags) { + vls_handle_t listen_vlsh, accept_vlsh; int rv; - const char *func_str; - u32 listen_sid = ldp_sid_from_fd (listen_fd); - int accept_sid; if ((errno = -ldp_init ())) return -1; - if (listen_sid != INVALID_SESSION_ID) + listen_vlsh = ldp_fd_to_vlsh (listen_fd); + if (listen_vlsh != VLS_INVALID_HANDLE) { vppcom_endpt_t ep; u8 src_addr[sizeof (struct sockaddr_in6)]; memset (&ep, 0, sizeof (ep)); ep.ip = src_addr; - func_str = "vppcom_session_accept"; + LDBG (0, "listen fd %d: calling vppcom_session_accept: listen sid %u," + " ep %p, flags 0x%x", listen_fd, listen_vlsh, &ep, flags); - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: listen fd %d (0x%x): calling %s(): " - "listen sid %u (0x%x), ep %p, flags 0x%x", - getpid (), listen_fd, listen_fd, func_str, - listen_sid, listen_sid, ep, flags); - - accept_sid = vppcom_session_accept (listen_sid, &ep, flags); - if (accept_sid < 0) + accept_vlsh = vls_accept (listen_vlsh, &ep, flags); + if (accept_vlsh < 0) { - errno = -accept_sid; + errno = -accept_vlsh; rv = -1; } else @@ -2844,56 +2144,26 @@ ldp_accept4 (int listen_fd, __SOCKADDR_ARG addr, rv = ldp_copy_ep_to_sockaddr (addr, addr_len, &ep); if (rv != VPPCOM_OK) { - (void) vppcom_session_close ((u32) accept_sid); + (void) vls_close (accept_vlsh); errno = -rv; rv = -1; } else { - func_str = "ldp_fd_from_sid"; - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: listen fd %d (0x%x): calling %s(): " - "accept sid %u (0x%x), ep %p, flags 0x%x", - getpid (), listen_fd, listen_fd, - func_str, accept_sid, accept_sid, ep, flags); - rv = ldp_fd_from_sid ((u32) accept_sid); - if (rv < 0) - { - (void) vppcom_session_close ((u32) accept_sid); - errno = -rv; - rv = -1; - } + rv = ldp_vlsh_to_fd (accept_vlsh); } } } else { - func_str = "libc_accept4"; - - if (LDP_DEBUG > 0) - clib_warning ("LDP<%d>: listen fd %d (0x%x): calling %s(): " - "addr %p, addr_len %p, flags 0x%x", - getpid (), listen_fd, listen_fd, func_str, - addr, addr_len, flags); + LDBG (0, "listen fd %d: calling libc_accept4(): addr %p, addr_len %p," + " flags 0x%x", listen_fd, addr, addr_len, flags); rv = libc_accept4 (listen_fd, addr, addr_len, flags); } - if (LDP_DEBUG > 0) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: listen fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), listen_fd, - listen_fd, func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: listen fd %d (0x%x): returning %d (0x%x)", - getpid (), listen_fd, listen_fd, rv, rv); - } + LDBG (1, "listen fd %d: accept returning %d", listen_fd, rv); + return rv; } @@ -2913,88 +2183,78 @@ accept (int fd, __SOCKADDR_ARG addr, socklen_t * __restrict addr_len) int shutdown (int fd, int how) { - int rv; - const char *func_str; - u32 sid = ldp_sid_from_fd (fd); + vls_handle_t vlsh; + int rv = 0, flags; + u32 flags_len = sizeof (flags); if ((errno = -ldp_init ())) return -1; - if (sid != INVALID_SESSION_ID) + vlsh = ldp_fd_to_vlsh (fd); + if (vlsh != VLS_INVALID_HANDLE) { - func_str = __func__; + LDBG (0, "called shutdown: fd %u vlsh %u how %d", fd, vlsh, how); - clib_warning ("LDP<%d>: LDP-TBD", getpid ()); - errno = ENOSYS; - rv = -1; + if (vls_attr (vlsh, VPPCOM_ATTR_SET_SHUT, &how, &flags_len)) + { + close (fd); + return -1; + } + + if (vls_attr (vlsh, VPPCOM_ATTR_GET_SHUT, &flags, &flags_len)) + { + close (fd); + return -1; + } + + if (flags == SHUT_RDWR) + rv = close (fd); } else { - func_str = "libc_shutdown"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: fd %d (0x%x): calling %s(): how %d", - getpid (), fd, fd, func_str, how); - + LDBG (0, "fd %d: calling libc_shutdown: how %d", fd, how); rv = libc_shutdown (fd, how); } - if (LDP_DEBUG > 1) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } int epoll_create1 (int flags) { - const char *func_str; + ldp_worker_ctx_t *ldpw = ldp_worker_get_current (); + vls_handle_t vlsh; int rv; if ((errno = -ldp_init ())) return -1; - func_str = "vppcom_epoll_create"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: calling %s()", getpid (), func_str); - - rv = vppcom_epoll_create (); + if (ldp->vcl_needs_real_epoll || vls_use_real_epoll ()) + { + /* Make sure workers have been allocated */ + if (!ldp->workers) + { + ldp_alloc_workers (); + ldpw = ldp_worker_get_current (); + } + rv = libc_epoll_create1 (flags); + ldp->vcl_needs_real_epoll = 0; + ldpw->vcl_mq_epfd = rv; + LDBG (0, "created vcl epfd %u", rv); + return rv; + } - if (PREDICT_FALSE (rv < 0)) + vlsh = vls_epoll_create (); + if (PREDICT_FALSE (vlsh == VLS_INVALID_HANDLE)) { - errno = -rv; + errno = -vlsh; rv = -1; } else - rv = ldp_fd_from_sid ((u32) rv); - - if (LDP_DEBUG > 1) { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: %s() failed! " - "rv %d, errno = %d", - getpid (), func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: returning epfd %d (0x%x)", getpid (), rv, rv); + rv = ldp_vlsh_to_fd (vlsh); } + LDBG (0, "epoll_create epfd %u vlsh %u", rv, vlsh); return rv; } @@ -3007,152 +2267,96 @@ epoll_create (int size) int epoll_ctl (int epfd, int op, int fd, struct epoll_event *event) { + vls_handle_t vep_vlsh, vlsh; int rv; - const char *func_str; - u32 vep_idx = ldp_sid_from_fd (epfd); if ((errno = -ldp_init ())) return -1; - if (PREDICT_TRUE (vep_idx != INVALID_SESSION_ID)) + vep_vlsh = ldp_fd_to_vlsh (epfd); + if (PREDICT_FALSE (vep_vlsh == VLS_INVALID_HANDLE)) { - u32 sid = ldp_sid_from_fd (fd); + /* The LDP epoll_create1 always creates VCL epfd's. + * The app should never have a kernel base epoll fd unless it + * was acquired outside of the LD_PRELOAD process context. + * In any case, if we get one, punt it to libc_epoll_ctl. + */ + LDBG (1, "epfd %d: calling libc_epoll_ctl: op %d, fd %d" + " event %p", epfd, op, fd, event); - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x), " - "sid %d (0x%x)", getpid (), epfd, epfd, - vep_idx, vep_idx, sid, sid); + rv = libc_epoll_ctl (epfd, op, fd, event); + goto done; + } - if (sid != INVALID_SESSION_ID) - { - func_str = "vppcom_epoll_ctl"; + vlsh = ldp_fd_to_vlsh (fd); - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "vep_idx %d (0x%x), op %d, sid %u (0x%x), event %p", - getpid (), epfd, epfd, func_str, vep_idx, vep_idx, - sid, sid, event); + LDBG (0, "epfd %d ep_vlsh %d, fd %u vlsh %d, op %u", epfd, vep_vlsh, fd, + vlsh, op); - rv = vppcom_epoll_ctl (vep_idx, op, sid, event); - if (rv != VPPCOM_OK) - { - errno = -rv; - rv = -1; - } - } - else + if (vlsh != VLS_INVALID_HANDLE) + { + LDBG (1, "epfd %d: calling vls_epoll_ctl: ep_vlsh %d op %d, vlsh %u," + " event %p", epfd, vep_vlsh, op, vlsh, event); + + rv = vls_epoll_ctl (vep_vlsh, op, vlsh, event); + if (rv != VPPCOM_OK) { - int libc_epfd; - u32 size = sizeof (epfd); + errno = -rv; + rv = -1; + } + } + else + { + int libc_epfd; + u32 size = sizeof (epfd); - func_str = "vppcom_session_attr[GET_LIBC_EPFD]"; - libc_epfd = vppcom_session_attr (vep_idx, - VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x): " - "%s() returned libc_epfd %d (0x%x)", - getpid (), epfd, epfd, vep_idx, vep_idx, - func_str, libc_epfd, libc_epfd); + libc_epfd = vls_attr (vep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); + if (!libc_epfd) + { + LDBG (1, "epfd %d, vep_vlsh %d calling libc_epoll_create1: " + "EPOLL_CLOEXEC", epfd, vep_vlsh); - if (!libc_epfd) + libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC); + if (libc_epfd < 0) { - func_str = "libc_epoll_create1"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x), vep_idx %d (0x%x): " - "calling %s(): EPOLL_CLOEXEC", - getpid (), epfd, epfd, vep_idx, vep_idx, - func_str); - - libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC); - if (libc_epfd < 0) - { - rv = libc_epfd; - goto done; - } - - func_str = "vppcom_session_attr[SET_LIBC_EPFD]"; - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "vep_idx %d (0x%x), VPPCOM_ATTR_SET_LIBC_EPFD, " - "libc_epfd %d (0x%x), size %d", - getpid (), epfd, epfd, func_str, - vep_idx, vep_idx, libc_epfd, libc_epfd, size); - - rv = vppcom_session_attr (vep_idx, VPPCOM_ATTR_SET_LIBC_EPFD, - &libc_epfd, &size); - if (rv < 0) - { - errno = -rv; - rv = -1; - goto done; - } + rv = libc_epfd; + goto done; } - else if (PREDICT_FALSE (libc_epfd < 0)) + + rv = vls_attr (vep_vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &libc_epfd, + &size); + if (rv < 0) { - errno = -epfd; + errno = -rv; rv = -1; goto done; } - - func_str = "libc_epoll_ctl"; - - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "libc_epfd %d (0x%x), op %d, " - "fd %d (0x%x), event %p", - getpid (), epfd, epfd, func_str, - libc_epfd, libc_epfd, op, fd, fd, event); - - rv = libc_epoll_ctl (libc_epfd, op, fd, event); } - } - else - { - /* The LDP epoll_create1 always creates VCL epfd's. - * The app should never have a kernel base epoll fd unless it - * was acquired outside of the LD_PRELOAD process context. - * In any case, if we get one, punt it to libc_epoll_ctl. - */ - func_str = "libc_epoll_ctl"; + else if (PREDICT_FALSE (libc_epfd < 0)) + { + errno = -epfd; + rv = -1; + goto done; + } - if (LDP_DEBUG > 1) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "op %d, fd %d (0x%x), event %p", - getpid (), epfd, epfd, func_str, op, fd, fd, event); + LDBG (1, "epfd %d: calling libc_epoll_ctl: libc_epfd %d, op %d, fd %d," + " event %p", epfd, libc_epfd, op, fd, event); - rv = libc_epoll_ctl (epfd, op, fd, event); + rv = libc_epoll_ctl (libc_epfd, op, fd, event); } done: - if (LDP_DEBUG > 1) - { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: fd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), fd, fd, - func_str, rv, errno_val); - errno = errno_val; - } - else - clib_warning ("LDP<%d>: fd %d (0x%x): returning %d (0x%x)", - getpid (), fd, fd, rv, rv); - } return rv; } static inline int -ldp_epoll_pwait (int epfd, struct epoll_event *events, - int maxevents, int timeout, const sigset_t * sigmask) +ldp_epoll_pwait (int epfd, struct epoll_event *events, int maxevents, + int timeout, const sigset_t * sigmask) { - const char *func_str; - int rv = 0; - double time_to_wait = (double) 0; - double time_out, now = 0; - u32 vep_idx = ldp_sid_from_fd (epfd); - int libc_epfd; + ldp_worker_ctx_t *ldpw = ldp_worker_get_current (); + double time_to_wait = (double) 0, max_time; + int libc_epfd, rv = 0; + vls_handle_t ep_vlsh; if ((errno = -ldp_init ())) return -1; @@ -3163,19 +2367,23 @@ ldp_epoll_pwait (int epfd, struct epoll_event *events, return -1; } - if (PREDICT_FALSE (vep_idx == INVALID_SESSION_ID)) + if (epfd == ldpw->vcl_mq_epfd) + return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask); + + ep_vlsh = ldp_fd_to_vlsh (epfd); + if (PREDICT_FALSE (ep_vlsh == VLS_INVALID_HANDLE)) { - clib_warning ("LDP<%d>: ERROR: epfd %d (0x%x): bad vep_idx %d (0x%x)!", - getpid (), epfd, epfd, vep_idx, vep_idx); + LDBG (0, "epfd %d: bad ep_vlsh %d!", epfd, ep_vlsh); errno = EBADFD; return -1; } - time_to_wait = ((timeout >= 0) ? (double) timeout : 0); - time_out = clib_time_now (&ldp->clib_time) + time_to_wait; + if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0)) + clib_time_init (&ldpw->clib_time); + time_to_wait = ((timeout >= 0) ? (double) timeout / 1000 : 0); + max_time = clib_time_now (&ldpw->clib_time) + time_to_wait; - func_str = "vppcom_session_attr[GET_LIBC_EPFD]"; - libc_epfd = vppcom_session_attr (vep_idx, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); + libc_epfd = vls_attr (ep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); if (PREDICT_FALSE (libc_epfd < 0)) { errno = -libc_epfd; @@ -3183,29 +2391,17 @@ ldp_epoll_pwait (int epfd, struct epoll_event *events, goto done; } - if (LDP_DEBUG > 2) - clib_warning ("LDP<%d>: epfd %d (0x%x): vep_idx %d (0x%x), " - "libc_epfd %d (0x%x), events %p, maxevents %d, " - "timeout %d, sigmask %p: time_to_wait %.02f", - getpid (), epfd, epfd, vep_idx, vep_idx, - libc_epfd, libc_epfd, events, maxevents, timeout, - sigmask, time_to_wait, time_out); + LDBG (2, "epfd %d: vep_idx %d, libc_epfd %d, events %p, maxevents %d, " + "timeout %d, sigmask %p: time_to_wait %.02f", epfd, ep_vlsh, + libc_epfd, events, maxevents, timeout, sigmask, time_to_wait); do { - if (!ldp->epoll_wait_vcl) + if (!ldpw->epoll_wait_vcl) { - func_str = "vppcom_epoll_wait"; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "vep_idx %d (0x%x), events %p, maxevents %d", - getpid (), epfd, epfd, func_str, - vep_idx, vep_idx, events, maxevents); - - rv = vppcom_epoll_wait (vep_idx, events, maxevents, 0); + rv = vls_epoll_wait (ep_vlsh, events, maxevents, 0); if (rv > 0) { - ldp->epoll_wait_vcl = 1; + ldpw->epoll_wait_vcl = 1; goto done; } else if (rv < 0) @@ -3216,47 +2412,132 @@ ldp_epoll_pwait (int epfd, struct epoll_event *events, } } else - ldp->epoll_wait_vcl = 0; + ldpw->epoll_wait_vcl = 0; if (libc_epfd > 0) { - func_str = "libc_epoll_pwait"; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: epfd %d (0x%x): calling %s(): " - "libc_epfd %d (0x%x), events %p, " - "maxevents %d, sigmask %p", - getpid (), epfd, epfd, func_str, - libc_epfd, libc_epfd, events, maxevents, sigmask); - - rv = libc_epoll_pwait (libc_epfd, events, maxevents, 1, sigmask); + rv = libc_epoll_pwait (libc_epfd, events, maxevents, 0, sigmask); if (rv != 0) goto done; } - - if (timeout != -1) - now = clib_time_now (&ldp->clib_time); } - while (now < time_out); + while ((timeout == -1) || (clib_time_now (&ldpw->clib_time) < max_time)); done: - if (LDP_DEBUG > 3) + return rv; +} + +static inline int +ldp_epoll_pwait_eventfd (int epfd, struct epoll_event *events, + int maxevents, int timeout, const sigset_t * sigmask) +{ + ldp_worker_ctx_t *ldpw; + int libc_epfd, rv = 0, num_ev; + vls_handle_t ep_vlsh; + + if ((errno = -ldp_init ())) + return -1; + + if (PREDICT_FALSE (!events || (timeout < -1))) { - if (libc_epfd > 0) - epfd = libc_epfd; + errno = EFAULT; + return -1; + } + + /* Make sure the vcl worker is valid. Could be that epoll fd was created on + * one thread but it is now used on another */ + if (PREDICT_FALSE (vppcom_worker_index () == ~0)) + vls_register_vcl_worker (); + + ldpw = ldp_worker_get_current (); + if (epfd == ldpw->vcl_mq_epfd) + return libc_epoll_pwait (epfd, events, maxevents, timeout, sigmask); + + ep_vlsh = ldp_fd_to_vlsh (epfd); + if (PREDICT_FALSE (ep_vlsh == VLS_INVALID_HANDLE)) + { + LDBG (0, "epfd %d: bad ep_vlsh %d!", epfd, ep_vlsh); + errno = EBADFD; + return -1; + } + + libc_epfd = vls_attr (ep_vlsh, VPPCOM_ATTR_GET_LIBC_EPFD, 0, 0); + if (PREDICT_FALSE (!libc_epfd)) + { + u32 size = sizeof (epfd); + + LDBG (1, "epfd %d, vep_vlsh %d calling libc_epoll_create1: " + "EPOLL_CLOEXEC", epfd, ep_vlsh); + libc_epfd = libc_epoll_create1 (EPOLL_CLOEXEC); + if (libc_epfd < 0) + { + rv = libc_epfd; + goto done; + } + + rv = vls_attr (ep_vlsh, VPPCOM_ATTR_SET_LIBC_EPFD, &libc_epfd, &size); if (rv < 0) { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: epfd %d (0x%x): %s() failed! " - "rv %d, errno = %d", getpid (), epfd, epfd, - func_str, rv, errno_val); - errno = errno_val; + errno = -rv; + rv = -1; + goto done; + } + } + if (PREDICT_FALSE (libc_epfd <= 0)) + { + errno = -libc_epfd; + rv = -1; + goto done; + } + + if (PREDICT_FALSE (!ldpw->mq_epfd_added)) + { + struct epoll_event e = { 0 }; + e.events = EPOLLIN; + e.data.fd = ldpw->vcl_mq_epfd; + if (libc_epoll_ctl (libc_epfd, EPOLL_CTL_ADD, ldpw->vcl_mq_epfd, &e) < + 0) + { + LDBG (0, "epfd %d, add libc mq epoll fd %d to libc epoll fd %d", + epfd, ldpw->vcl_mq_epfd, libc_epfd); + rv = -1; + goto done; + } + ldpw->mq_epfd_added = 1; + } + + rv = vls_epoll_wait (ep_vlsh, events, maxevents, 0); + if (rv > 0) + goto done; + else if (PREDICT_FALSE (rv < 0)) + { + errno = -rv; + rv = -1; + goto done; + } + + rv = libc_epoll_pwait (libc_epfd, events, maxevents, timeout, sigmask); + if (rv <= 0) + goto done; + for (int i = 0; i < rv; i++) + { + if (events[i].data.fd == ldpw->vcl_mq_epfd) + { + /* We should remove mq epoll fd from events. */ + rv--; + if (i != rv) + { + events[i].events = events[rv].events; + events[i].data.u64 = events[rv].data.u64; + } + num_ev = vls_epoll_wait (ep_vlsh, &events[rv], maxevents - rv, 0); + if (PREDICT_TRUE (num_ev > 0)) + rv += num_ev; + break; } - else - clib_warning ("LDP<%d>: epfd %d (0x%x): returning %d (0x%x)", - getpid (), epfd, epfd, rv, rv); } + +done: return rv; } @@ -3264,80 +2545,72 @@ int epoll_pwait (int epfd, struct epoll_event *events, int maxevents, int timeout, const sigset_t * sigmask) { - return ldp_epoll_pwait (epfd, events, maxevents, timeout, sigmask); + if (vls_use_eventfd ()) + return ldp_epoll_pwait_eventfd (epfd, events, maxevents, timeout, + sigmask); + else + return ldp_epoll_pwait (epfd, events, maxevents, timeout, sigmask); } int epoll_wait (int epfd, struct epoll_event *events, int maxevents, int timeout) { - return ldp_epoll_pwait (epfd, events, maxevents, timeout, NULL); + if (vls_use_eventfd ()) + return ldp_epoll_pwait_eventfd (epfd, events, maxevents, timeout, NULL); + else + return ldp_epoll_pwait (epfd, events, maxevents, timeout, NULL); } int poll (struct pollfd *fds, nfds_t nfds, int timeout) { - const char *func_str = __func__; - int rv, i, n_libc_fds, n_revents; - u32 sid; + ldp_worker_ctx_t *ldpw = ldp_worker_get_current (); + int rv, i, n_revents = 0; + vls_handle_t vlsh; vcl_poll_t *vp; - double wait_for_time; + double max_time; - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: fds %p, nfds %d, timeout %d", - getpid (), fds, nfds, timeout); + LDBG (3, "fds %p, nfds %ld, timeout %d", fds, nfds, timeout); - if (timeout >= 0) - wait_for_time = (f64) timeout / 1000; - else - wait_for_time = -1; + if (PREDICT_FALSE (ldpw->clib_time.init_cpu_time == 0)) + clib_time_init (&ldpw->clib_time); + + max_time = (timeout >= 0) ? (f64) timeout / 1000 : 0; + max_time += clib_time_now (&ldpw->clib_time); - n_libc_fds = 0; for (i = 0; i < nfds; i++) { - if (fds[i].fd >= 0) - { - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: fds[%d].fd %d (0x%0x), .events = 0x%x, " - ".revents = 0x%x", getpid (), i, fds[i].fd, - fds[i].fd, fds[i].events, fds[i].revents); + if (fds[i].fd < 0) + continue; - sid = ldp_sid_from_fd (fds[i].fd); - if (sid != INVALID_SESSION_ID) - { - fds[i].fd = -fds[i].fd; - vec_add2 (ldp->vcl_poll, vp, 1); - vp->fds_ndx = i; - vp->sid = sid; - vp->events = fds[i].events; + vlsh = ldp_fd_to_vlsh (fds[i].fd); + if (vlsh != VLS_INVALID_HANDLE) + { + fds[i].fd = -fds[i].fd; + vec_add2 (ldpw->vcl_poll, vp, 1); + vp->fds_ndx = i; + vp->sh = vlsh_to_sh (vlsh); + vp->events = fds[i].events; #ifdef __USE_XOPEN2K - if (fds[i].events & POLLRDNORM) - vp->events |= POLLIN; - if (fds[i].events & POLLWRNORM) - vp->events |= POLLOUT; + if (fds[i].events & POLLRDNORM) + vp->events |= POLLIN; + if (fds[i].events & POLLWRNORM) + vp->events |= POLLOUT; #endif - vp->revents = &fds[i].revents; - } - else - n_libc_fds++; + vp->revents = fds[i].revents; + } + else + { + vec_add1 (ldpw->libc_poll, fds[i]); + vec_add1 (ldpw->libc_poll_idxs, i); } } - n_revents = 0; do { - if (vec_len (ldp->vcl_poll)) + if (vec_len (ldpw->vcl_poll)) { - func_str = "vppcom_poll"; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: calling %s(): " - "vcl_poll %p, n_sids %u (0x%x): " - "n_libc_fds %u", - getpid (), func_str, ldp->vcl_poll, - vec_len (ldp->vcl_poll), vec_len (ldp->vcl_poll), - n_libc_fds); - - rv = vppcom_poll (ldp->vcl_poll, vec_len (ldp->vcl_poll), 0); + rv = vppcom_poll (ldpw->vcl_poll, vec_len (ldpw->vcl_poll), 0); if (rv < 0) { errno = -rv; @@ -3348,15 +2621,9 @@ poll (struct pollfd *fds, nfds_t nfds, int timeout) n_revents += rv; } - if (n_libc_fds) + if (vec_len (ldpw->libc_poll)) { - func_str = "libc_poll"; - - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: calling %s(): fds %p, nfds %u: n_sids %u", - getpid (), fds, nfds, vec_len (ldp->vcl_poll)); - - rv = libc_poll (fds, nfds, 0); + rv = libc_poll (ldpw->libc_poll, vec_len (ldpw->libc_poll), 0); if (rv < 0) goto done; else @@ -3369,14 +2636,14 @@ poll (struct pollfd *fds, nfds_t nfds, int timeout) goto done; } } - while ((wait_for_time == -1) || - (clib_time_now (&ldp->clib_time) < wait_for_time)); + while ((timeout < 0) || (clib_time_now (&ldpw->clib_time) < max_time)); rv = 0; done: - vec_foreach (vp, ldp->vcl_poll) + vec_foreach (vp, ldpw->vcl_poll) { fds[vp->fds_ndx].fd = -fds[vp->fds_ndx].fd; + fds[vp->fds_ndx].revents = vp->revents; #ifdef __USE_XOPEN2K if ((fds[vp->fds_ndx].revents & POLLIN) && (fds[vp->fds_ndx].events & POLLRDNORM)) @@ -3386,38 +2653,14 @@ done: fds[vp->fds_ndx].revents |= POLLWRNORM; #endif } - vec_reset_length (ldp->vcl_poll); + vec_reset_length (ldpw->vcl_poll); - if (LDP_DEBUG > 3) + for (i = 0; i < vec_len (ldpw->libc_poll); i++) { - if (rv < 0) - { - int errno_val = errno; - perror (func_str); - clib_warning ("LDP<%d>: ERROR: %s() failed! " - "rv %d, errno = %d", getpid (), - func_str, rv, errno_val); - errno = errno_val; - } - else - { - clib_warning ("LDP<%d>: returning %d (0x%x): n_sids %u, " - "n_libc_fds %d", getpid (), rv, rv, - vec_len (ldp->vcl_poll), n_libc_fds); - - for (i = 0; i < nfds; i++) - { - if (fds[i].fd >= 0) - { - if (LDP_DEBUG > 3) - clib_warning ("LDP<%d>: fds[%d].fd %d (0x%0x), " - ".events = 0x%x, .revents = 0x%x", - getpid (), i, fds[i].fd, fds[i].fd, - fds[i].events, fds[i].revents); - } - } - } + fds[ldpw->libc_poll_idxs[i]].revents = ldpw->libc_poll[i].revents; } + vec_reset_length (ldpw->libc_poll_idxs); + vec_reset_length (ldpw->libc_poll); return rv; } @@ -3450,8 +2693,11 @@ ldp_constructor (void) { swrap_constructor (); if (ldp_init () != 0) - fprintf (stderr, "\nLDP<%d>: ERROR: ldp_constructor: failed!\n", - getpid ()); + { + fprintf (stderr, "\nLDP<%d>: ERROR: ldp_constructor: failed!\n", + getpid ()); + _exit (1); + } else if (LDP_DEBUG > 0) clib_warning ("LDP<%d>: LDP constructor: done!\n", getpid ()); } @@ -3462,19 +2708,18 @@ ldp_constructor (void) void ldp_destructor (void) { - swrap_destructor (); - if (ldp->init) - { - vppcom_app_destroy (); - ldp->init = 0; - } + /* + swrap_destructor (); + if (ldp->init) + ldp->init = 0; + */ /* Don't use clib_warning() here because that calls writev() * which will call ldp_init(). */ if (LDP_DEBUG > 0) - printf ("%s:%d: LDP<%d>: LDP destructor: done!\n", - __func__, __LINE__, getpid ()); + fprintf (stderr, "%s:%d: LDP<%d>: LDP destructor: done!\n", + __func__, __LINE__, getpid ()); }