From edda5926e0b755368aef017a278272b796d22467 Mon Sep 17 00:00:00 2001 From: Florin Coras Date: Wed, 28 Apr 2021 13:02:33 -0700 Subject: [PATCH] hsa: vcl test perf improvements - poll session events in server - init session buffers based on config - cleanup some of the data structures Type: improvement Signed-off-by: Florin Coras Change-Id: I81c19e6546c8292db07b63c66e4da03ef7f55e22 --- src/plugins/hs_apps/vcl/vcl_test.h | 24 ++++-- src/plugins/hs_apps/vcl/vcl_test_client.c | 31 +++----- src/plugins/hs_apps/vcl/vcl_test_server.c | 126 ++++++++++++++++-------------- 3 files changed, 93 insertions(+), 88 deletions(-) diff --git a/src/plugins/hs_apps/vcl/vcl_test.h b/src/plugins/hs_apps/vcl/vcl_test.h index 18e3c00f487..d475ec3c0a2 100644 --- a/src/plugins/hs_apps/vcl/vcl_test.h +++ b/src/plugins/hs_apps/vcl/vcl_test.h @@ -126,6 +126,7 @@ typedef struct vcl_test_session { uint8_t is_alloc; uint8_t is_open; + uint8_t is_done; int fd; int (*read) (struct vcl_test_session *ts, void *buf, uint32_t buflen); int (*write) (struct vcl_test_session *ts, void *buf, uint32_t buflen); @@ -247,14 +248,23 @@ vcl_test_buf_alloc (vcl_test_cfg_t * cfg, uint8_t is_rxbuf, uint8_t ** buf, } static inline void -vcl_test_session_buf_alloc (vcl_test_session_t * socket) +vcl_test_session_buf_alloc (vcl_test_session_t *ts) { - socket->rxbuf_size = socket->cfg.rxbuf_size; - socket->txbuf_size = socket->cfg.txbuf_size; - vcl_test_buf_alloc (&socket->cfg, 0 /* is_rxbuf */ , - (uint8_t **) & socket->txbuf, &socket->txbuf_size); - vcl_test_buf_alloc (&socket->cfg, 1 /* is_rxbuf */ , - (uint8_t **) & socket->rxbuf, &socket->rxbuf_size); + ts->rxbuf_size = ts->cfg.rxbuf_size; + ts->txbuf_size = ts->cfg.txbuf_size; + vcl_test_buf_alloc (&ts->cfg, 0 /* is_rxbuf */, (uint8_t **) &ts->txbuf, + &ts->txbuf_size); + vcl_test_buf_alloc (&ts->cfg, 1 /* is_rxbuf */, (uint8_t **) &ts->rxbuf, + &ts->rxbuf_size); +} + +static inline void +vcl_test_session_buf_free (vcl_test_session_t *ts) +{ + free (ts->rxbuf); + free (ts->txbuf); + ts->rxbuf = 0; + ts->txbuf = 0; } static inline char * diff --git a/src/plugins/hs_apps/vcl/vcl_test_client.c b/src/plugins/hs_apps/vcl/vcl_test_client.c index 83d6dd057f6..1f2c47dac37 100644 --- a/src/plugins/hs_apps/vcl/vcl_test_client.c +++ b/src/plugins/hs_apps/vcl/vcl_test_client.c @@ -31,7 +31,6 @@ typedef struct vcl_test_session_t *sessions; vcl_test_session_t *qsessions; uint32_t n_sessions; - uint32_t n_qsessions; uint32_t wrk_index; fd_set wr_fdset; fd_set rd_fdset; @@ -43,10 +42,9 @@ typedef struct typedef struct { vcl_test_client_worker_t *workers; + vcl_test_session_t ctrl_session; vppcom_endpt_t server_endpt; uint32_t cfg_seq_num; - vcl_test_session_t ctrl_session; - vcl_test_session_t *sessions; uint8_t dump_cfg; vcl_test_t post_test; uint8_t proto; @@ -154,7 +152,10 @@ vtc_connect_test_sessions (vcl_test_client_worker_t * wrk) for (i = 0; i < n_test_sessions; i++) { ts = &wrk->sessions[i]; + memset (ts, 0, sizeof (*ts)); ts->session_index = i; + ts->cfg = wrk->cfg; + vcl_test_session_buf_alloc (ts); rv = tp->open (&wrk->sessions[i], &vcm->server_endpt); if (rv < 0) return rv; @@ -169,8 +170,6 @@ done: static int vtc_worker_test_setup (vcl_test_client_worker_t * wrk) { - vcl_test_client_main_t *vcm = &vcl_client_main; - vcl_test_session_t *ctrl = &vcm->ctrl_session; vcl_test_cfg_t *cfg = &wrk->cfg; vcl_test_session_t *ts; uint32_t sidx; @@ -182,19 +181,16 @@ vtc_worker_test_setup (vcl_test_client_worker_t * wrk) for (i = 0; i < cfg->num_test_sessions; i++) { ts = &wrk->sessions[i]; - ts->cfg = wrk->cfg; - vcl_test_session_buf_alloc (ts); switch (cfg->test) { - case VCL_TEST_TYPE_ECHO: - memcpy (ts->txbuf, ctrl->txbuf, cfg->total_bytes); - break; case VCL_TEST_TYPE_UNI: case VCL_TEST_TYPE_BI: for (j = 0; j < ts->txbuf_size; j++) ts->txbuf[j] = j & 0xff; break; + default: + break; } FD_SET (vppcom_session_index (ts->fd), &wrk->wr_fdset); @@ -211,9 +207,6 @@ static int vtc_worker_init (vcl_test_client_worker_t * wrk) { vcl_test_client_main_t *vcm = &vcl_client_main; - vcl_test_cfg_t *cfg = &wrk->cfg; - vcl_test_session_t *ts; - uint32_t n; int rv; __wrk_index = wrk->wrk_index; @@ -239,12 +232,6 @@ vtc_worker_init (vcl_test_client_worker_t * wrk) if (vtc_worker_test_setup (wrk)) return -1; - for (n = 0; n < cfg->num_test_sessions; n++) - { - ts = &wrk->sessions[n]; - memset (&ts->stats, 0, sizeof (ts->stats)); - } - return 0; } @@ -295,6 +282,7 @@ vtc_worker_sessions_exit (vcl_test_client_worker_t * wrk) { ts = &wrk->sessions[i]; vppcom_session_close (ts->fd); + vcl_test_session_buf_free (ts); } wrk->n_sessions = 0; @@ -344,8 +332,7 @@ vtc_worker_loop (void *arg) for (i = 0; i < wrk->cfg.num_test_sessions; i++) { ts = &wrk->sessions[i]; - if (!((ts->stats.stop.tv_sec == 0) && - (ts->stats.stop.tv_nsec == 0))) + if (ts->is_done) continue; if (FD_ISSET (vppcom_session_index (ts->fd), rfdset) @@ -371,6 +358,7 @@ vtc_worker_loop (void *arg) || (check_rx && ts->stats.rx_bytes >= ts->cfg.total_bytes)) { clock_gettime (CLOCK_REALTIME, &ts->stats.stop); + ts->is_done = 1; n_active_sessions--; } } @@ -454,6 +442,7 @@ vtc_stream_client (vcl_test_client_main_t * vcm) vtinf ("%s-directional Stream Test Starting!", ctrl->cfg.test == VCL_TEST_TYPE_BI ? "Bi" : "Uni"); + memset (&ctrl->stats, 0, sizeof (vcl_test_stats_t)); cfg->total_bytes = cfg->num_writes * cfg->txbuf_size; cfg->ctrl_handle = ctrl->fd; diff --git a/src/plugins/hs_apps/vcl/vcl_test_server.c b/src/plugins/hs_apps/vcl/vcl_test_server.c index ab0c6014620..a36801b0691 100644 --- a/src/plugins/hs_apps/vcl/vcl_test_server.c +++ b/src/plugins/hs_apps/vcl/vcl_test_server.c @@ -39,21 +39,20 @@ typedef struct typedef struct { + vcl_test_session_t *conn_pool; uint32_t wrk_index; - vcl_test_session_t listener; int epfd; - struct epoll_event wait_events[VCL_TEST_CFG_MAX_EPOLL_EVENTS]; - size_t conn_pool_size; - vcl_test_session_t *conn_pool; + int conn_pool_size; int nfds; + vcl_test_session_t listener; pthread_t thread_handle; } vcl_test_server_worker_t; typedef struct { - vcl_test_server_cfg_t cfg; vcl_test_server_worker_t *workers; vcl_test_session_t *ctrl; + vcl_test_server_cfg_t server_cfg; int ctrl_listen_fd; struct sockaddr_storage servaddr; volatile int worker_fails; @@ -79,10 +78,6 @@ conn_pool_expand (vcl_test_server_worker_t * wrk, size_t expand_size) { vcl_test_session_t *conn = &conn_pool[i]; memset (conn, 0, sizeof (*conn)); - vcl_test_cfg_init (&conn->cfg); - vcl_test_buf_alloc (&conn->cfg, 1 /* is_rxbuf */, - (uint8_t **) &conn->rxbuf, &conn->rxbuf_size); - conn->cfg.txbuf_size = conn->cfg.rxbuf_size; } wrk->conn_pool = conn_pool; @@ -97,6 +92,7 @@ conn_pool_expand (vcl_test_server_worker_t * wrk, size_t expand_size) static inline vcl_test_session_t * conn_pool_alloc (vcl_test_server_worker_t *wrk) { + vcl_test_session_t *conn; int i, expand = 0; again: @@ -104,9 +100,12 @@ again: { if (!wrk->conn_pool[i].is_alloc) { - wrk->conn_pool[i].endpt.ip = wrk->conn_pool[i].ip; - wrk->conn_pool[i].is_alloc = 1; - wrk->conn_pool[i].session_index = i; + conn = &wrk->conn_pool[i]; + conn->endpt.ip = wrk->conn_pool[i].ip; + conn->is_alloc = 1; + conn->session_index = i; + memset (&conn->stats, 0, sizeof (vcl_test_stats_t)); + vcl_test_cfg_init (&conn->cfg); return (&wrk->conn_pool[i]); } } @@ -122,10 +121,11 @@ again: } static inline void -conn_pool_free (vcl_test_session_t *conn) +conn_pool_free (vcl_test_session_t *ts) { - conn->fd = 0; - conn->is_alloc = 0; + ts->fd = 0; + ts->is_alloc = 0; + vcl_test_session_buf_free (ts); } static inline void @@ -153,18 +153,18 @@ vts_session_close (vcl_test_session_t *conn) if (!conn->is_open) return; - if (vt->protos[vsm->cfg.proto]->close) - vt->protos[vsm->cfg.proto]->close (conn); + if (vt->protos[vsm->server_cfg.proto]->close) + vt->protos[vsm->server_cfg.proto]->close (conn); vppcom_session_close (conn->fd); conn->is_open = 0; } static void -vts_session_cleanup (vcl_test_session_t *conn) +vts_session_cleanup (vcl_test_session_t *ts) { - vts_session_close (conn); - conn_pool_free (conn); + vts_session_close (ts); + conn_pool_free (ts); } static void @@ -214,7 +214,7 @@ vts_test_cmd (vcl_test_server_worker_t *wrk, vcl_test_session_t *conn, vcl_test_stats_accumulate (&conn->stats, &tc->stats); if (tc->is_open) { - vts_session_close (tc); + vts_session_cleanup (tc); continue; } /* Only relevant if all connections previously closed */ @@ -321,7 +321,11 @@ vts_accept_client (vcl_test_server_worker_t *wrk, int listen_fd) return 0; } - tp = vcl_test_main.protos[vsm->cfg.proto]; + if (vsm->ctrl) + conn->cfg = vsm->ctrl->cfg; + vcl_test_session_buf_alloc (conn); + + tp = vcl_test_main.protos[vsm->server_cfg.proto]; if (tp->accept (listen_fd, conn)) return 0; @@ -362,34 +366,34 @@ vcl_test_init_endpoint_addr (vcl_test_server_main_t * vsm) struct sockaddr_storage *servaddr = &vsm->servaddr; memset (servaddr, 0, sizeof (*servaddr)); - if (vsm->cfg.address_ip6) + if (vsm->server_cfg.address_ip6) { struct sockaddr_in6 *server_addr = (struct sockaddr_in6 *) servaddr; server_addr->sin6_family = AF_INET6; server_addr->sin6_addr = in6addr_any; - server_addr->sin6_port = htons (vsm->cfg.port); + server_addr->sin6_port = htons (vsm->server_cfg.port); } else { struct sockaddr_in *server_addr = (struct sockaddr_in *) servaddr; server_addr->sin_family = AF_INET; server_addr->sin_addr.s_addr = htonl (INADDR_ANY); - server_addr->sin_port = htons (vsm->cfg.port); + server_addr->sin_port = htons (vsm->server_cfg.port); } - if (vsm->cfg.address_ip6) + if (vsm->server_cfg.address_ip6) { struct sockaddr_in6 *server_addr = (struct sockaddr_in6 *) servaddr; - vsm->cfg.endpt.is_ip4 = 0; - vsm->cfg.endpt.ip = (uint8_t *) & server_addr->sin6_addr; - vsm->cfg.endpt.port = (uint16_t) server_addr->sin6_port; + vsm->server_cfg.endpt.is_ip4 = 0; + vsm->server_cfg.endpt.ip = (uint8_t *) &server_addr->sin6_addr; + vsm->server_cfg.endpt.port = (uint16_t) server_addr->sin6_port; } else { struct sockaddr_in *server_addr = (struct sockaddr_in *) servaddr; - vsm->cfg.endpt.is_ip4 = 1; - vsm->cfg.endpt.ip = (uint8_t *) & server_addr->sin_addr; - vsm->cfg.endpt.port = (uint16_t) server_addr->sin_port; + vsm->server_cfg.endpt.is_ip4 = 1; + vsm->server_cfg.endpt.ip = (uint8_t *) &server_addr->sin_addr; + vsm->server_cfg.endpt.port = (uint16_t) server_addr->sin_port; } } @@ -399,33 +403,33 @@ vcl_test_server_process_opts (vcl_test_server_main_t * vsm, int argc, { int v, c; - vsm->cfg.proto = VPPCOM_PROTO_TCP; + vsm->server_cfg.proto = VPPCOM_PROTO_TCP; opterr = 0; while ((c = getopt (argc, argv, "6DLsw:hp:")) != -1) switch (c) { case '6': - vsm->cfg.address_ip6 = 1; + vsm->server_cfg.address_ip6 = 1; break; case 'p': - if (vppcom_unformat_proto (&vsm->cfg.proto, optarg)) + if (vppcom_unformat_proto (&vsm->server_cfg.proto, optarg)) vtwrn ("Invalid vppcom protocol %s, defaulting to TCP", optarg); break; case 'D': - vsm->cfg.proto = VPPCOM_PROTO_UDP; + vsm->server_cfg.proto = VPPCOM_PROTO_UDP; break; case 'L': - vsm->cfg.proto = VPPCOM_PROTO_TLS; + vsm->server_cfg.proto = VPPCOM_PROTO_TLS; break; case 'w': v = atoi (optarg); if (v > 1) - vsm->cfg.workers = v; + vsm->server_cfg.workers = v; else vtwrn ("Invalid number of workers %d", v); break; @@ -458,7 +462,7 @@ vcl_test_server_process_opts (vcl_test_server_main_t * vsm, int argc, } if (sscanf (argv[optind], "%d", &v) == 1) - vsm->cfg.port = (uint16_t) v; + vsm->server_cfg.port = (uint16_t) v; else { fprintf (stderr, "SERVER: ERROR: Invalid port (%s)!\n", argv[optind]); @@ -540,8 +544,8 @@ vts_worker_init (vcl_test_server_worker_t * wrk) if (vppcom_worker_register ()) vtfail ("vppcom_worker_register()", 1); - tp = vt->protos[vsm->cfg.proto]; - if ((rv = tp->listen (&wrk->listener, &vsm->cfg.endpt))) + tp = vt->protos[vsm->server_cfg.proto]; + if ((rv = tp->listen (&wrk->listener, &vsm->server_cfg.endpt))) vtfail ("proto listen", rv); /* First worker already has epoll fd */ @@ -561,7 +565,7 @@ vts_worker_init (vcl_test_server_worker_t * wrk) vsm->active_workers += 1; vtinf ("Waiting for client data connections on port %d ...", - ntohs (vsm->cfg.endpt.port)); + ntohs (vsm->server_cfg.endpt.port)); } static inline int @@ -577,6 +581,7 @@ vts_conn_read (vcl_test_session_t *conn) static void * vts_worker_loop (void *arg) { + struct epoll_event ep_evts[VCL_TEST_CFG_MAX_EPOLL_EVENTS]; vcl_test_server_main_t *vsm = &vcl_server_main; vcl_test_server_worker_t *wrk = arg; vcl_test_session_t *conn; @@ -588,8 +593,9 @@ vts_worker_loop (void *arg) while (1) { - num_ev = vppcom_epoll_wait (wrk->epfd, wrk->wait_events, - VCL_TEST_CFG_MAX_EPOLL_EVENTS, 60000.0); + num_ev = + vppcom_epoll_wait (wrk->epfd, ep_evts, VCL_TEST_CFG_MAX_EPOLL_EVENTS, + 0 /* poll session events */); if (num_ev < 0) { vterr ("vppcom_epoll_wait()", num_ev); @@ -597,18 +603,17 @@ vts_worker_loop (void *arg) } else if (num_ev == 0) { - vtinf ("vppcom_epoll_wait() timeout!"); continue; } for (i = 0; i < num_ev; i++) { - conn = &wrk->conn_pool[wrk->wait_events[i].data.u32]; + conn = &wrk->conn_pool[ep_evts[i].data.u32]; /* * Check for close events */ - if (wrk->wait_events[i].events & (EPOLLHUP | EPOLLRDHUP)) + if (ep_evts[i].events & (EPOLLHUP | EPOLLRDHUP)) { - vts_session_close (conn); + vts_session_cleanup (conn); wrk->nfds--; if (!wrk->nfds) { @@ -622,7 +627,7 @@ vts_worker_loop (void *arg) * Check if new session needs to be accepted */ - if (wrk->wait_events[i].data.u32 == VCL_TEST_CTRL_LISTENER) + if (!wrk->wrk_index && ep_evts[i].data.u32 == VCL_TEST_CTRL_LISTENER) { if (vsm->ctrl) { @@ -632,7 +637,7 @@ vts_worker_loop (void *arg) vsm->ctrl = vts_accept_client (wrk, vsm->ctrl_listen_fd); continue; } - if (wrk->wait_events[i].data.u32 == VCL_TEST_DATA_LISTENER) + if (ep_evts[i].data.u32 == VCL_TEST_DATA_LISTENER) { conn = vts_accept_client (wrk, wrk->listener.fd); conn->cfg = vsm->ctrl->cfg; @@ -676,7 +681,7 @@ vts_worker_loop (void *arg) * Read perf test data */ - if (EPOLLIN & wrk->wait_events[i].events) + if (EPOLLIN & ep_evts[i].events) { read_again: rx_bytes = vts_conn_read (conn); @@ -730,7 +735,7 @@ vts_ctrl_session_init (vcl_test_server_worker_t *wrk) if (vsm->ctrl_listen_fd < 0) vtfail ("vppcom_session_create()", vsm->ctrl_listen_fd); - rv = vppcom_session_bind (vsm->ctrl_listen_fd, &vsm->cfg.endpt); + rv = vppcom_session_bind (vsm->ctrl_listen_fd, &vsm->server_cfg.endpt); if (rv < 0) vtfail ("vppcom_session_bind()", rv); @@ -749,7 +754,8 @@ vts_ctrl_session_init (vcl_test_server_worker_t *wrk) if (rv < 0) vtfail ("vppcom_epoll_ctl", rv); - vtinf ("Waiting for client ctrl connection on port %d ...", vsm->cfg.port); + vtinf ("Waiting for client ctrl connection on port %d ...", + vsm->server_cfg.port); } int @@ -760,8 +766,8 @@ main (int argc, char **argv) int rv, i; clib_mem_init_thread_safe (0, 64 << 20); - vsm->cfg.port = VCL_TEST_SERVER_PORT; - vsm->cfg.workers = 1; + vsm->server_cfg.port = VCL_TEST_SERVER_PORT; + vsm->server_cfg.workers = 1; vsm->active_workers = 0; vcl_test_server_process_opts (vsm, argc, argv); @@ -770,16 +776,16 @@ main (int argc, char **argv) vtfail ("vppcom_app_create()", rv); /* Protos like tls/dtls/quic need init */ - if (vt->protos[vsm->cfg.proto]->init) - vt->protos[vsm->cfg.proto]->init (0); + if (vt->protos[vsm->server_cfg.proto]->init) + vt->protos[vsm->server_cfg.proto]->init (0); - vsm->workers = calloc (vsm->cfg.workers, sizeof (*vsm->workers)); + vsm->workers = calloc (vsm->server_cfg.workers, sizeof (*vsm->workers)); vts_ctrl_session_init (&vsm->workers[0]); /* Update ctrl port to data port */ - vsm->cfg.endpt.port += 1; + vsm->server_cfg.endpt.port += 1; vts_worker_init (&vsm->workers[0]); - for (i = 1; i < vsm->cfg.workers; i++) + for (i = 1; i < vsm->server_cfg.workers; i++) { vsm->workers[i].wrk_index = i; rv = pthread_create (&vsm->workers[i].thread_handle, NULL, -- 2.16.6