From a2b7a02d9c3e03b4923f4153714f43e361960cf0 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Tue, 28 Dec 2021 20:32:20 +0100 Subject: [PATCH] vlib: remove external thread management support Now DPDK have API to register external threads so we can remove this mess... Type: improvement Change-Id: I71a21f0cd94bd668aa406710c75a0bcc63fdc840 Signed-off-by: Damjan Marion --- src/plugins/dpdk/CMakeLists.txt | 1 - src/plugins/dpdk/device/cli.c | 3 +- src/plugins/dpdk/device/dpdk.h | 24 +---------- src/plugins/dpdk/device/dpdk_priv.h | 4 -- src/plugins/dpdk/device/init.c | 75 +++++--------------------------- src/plugins/dpdk/thread.c | 85 ------------------------------------- src/vlib/threads.c | 42 +++--------------- src/vlib/threads.h | 17 -------- 8 files changed, 19 insertions(+), 232 deletions(-) delete mode 100644 src/plugins/dpdk/thread.c diff --git a/src/plugins/dpdk/CMakeLists.txt b/src/plugins/dpdk/CMakeLists.txt index 5de75e76289..b53f4ad2375 100644 --- a/src/plugins/dpdk/CMakeLists.txt +++ b/src/plugins/dpdk/CMakeLists.txt @@ -131,7 +131,6 @@ add_vpp_plugin(dpdk SOURCES buffer.c main.c - thread.c device/cli.c device/common.c device/device.c diff --git a/src/plugins/dpdk/device/cli.c b/src/plugins/dpdk/device/cli.c index 0f771c6ba77..ba922e29a54 100644 --- a/src/plugins/dpdk/device/cli.c +++ b/src/plugins/dpdk/device/cli.c @@ -373,7 +373,8 @@ show_dpdk_version_command_fn (vlib_main_t * vm, * Example of how to display how many DPDK buffer test command has allocated: * @cliexstart{show dpdk version} * DPDK Version: DPDK 16.11.0 - * DPDK EAL init args: -c 1 -n 4 --huge-dir /run/vpp/hugepages --file-prefix vpp -w 0000:00:08.0 -w 0000:00:09.0 --master-lcore 0 --socket-mem 256 + * DPDK EAL init args: --in-memory --no-telemetry --file-prefix vpp + * -w 0000:00:08.0 -w 0000:00:09.0 * @cliexend ?*/ /* *INDENT-OFF* */ diff --git a/src/plugins/dpdk/device/dpdk.h b/src/plugins/dpdk/device/dpdk.h index 51f80a80462..cc7305dbdc4 100644 --- a/src/plugins/dpdk/device/dpdk.h +++ b/src/plugins/dpdk/device/dpdk.h @@ -22,27 +22,12 @@ #include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include +#include #include -#include #include -#include -#include -#include #include -#include #include -#include -#include #include @@ -313,13 +298,6 @@ typedef struct #define DPDK_MAX_SIMD_BITWIDTH_256 256 #define DPDK_MAX_SIMD_BITWIDTH_512 512 - /* Required config parameters */ - u8 coremask_set_manually; - u8 nchannels_set_manually; - u32 coremask; - u32 nchannels; - u32 num_crypto_mbufs; - /* * format interface names ala xxxEthernet%d/%d/%d instead of * xxxEthernet%x/%x/%x. diff --git a/src/plugins/dpdk/device/dpdk_priv.h b/src/plugins/dpdk/device/dpdk_priv.h index 7f595ae8e19..267c0b55aa3 100644 --- a/src/plugins/dpdk/device/dpdk_priv.h +++ b/src/plugins/dpdk/device/dpdk_priv.h @@ -33,10 +33,6 @@ _(no-hpet) \ _(no-huge) \ _(vmware-tsc-map) -#define foreach_eal_single_hyphen_mandatory_arg \ -_(coremask, c) \ -_(nchannels, n) \ - #define foreach_eal_single_hyphen_arg \ _(mem-alloc-request, m) \ _(force-ranks, r) diff --git a/src/plugins/dpdk/device/init.c b/src/plugins/dpdk/device/init.c index 3808a5b1e4d..04d6f89ac7b 100644 --- a/src/plugins/dpdk/device/init.c +++ b/src/plugins/dpdk/device/init.c @@ -1504,11 +1504,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) num_whitelisted++; } - else if (unformat (input, "num-mem-channels %d", &conf->nchannels)) - conf->nchannels_set_manually = 0; - else if (unformat (input, "num-crypto-mbufs %d", - &conf->num_crypto_mbufs)) - ; else if (unformat (input, "uio-driver %s", &conf->uio_driver_name)) ; else if (unformat (input, "socket-mem %s", &socket_mem)) @@ -1584,17 +1579,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) vec_add1 (conf->eal_init_args, s); \ } foreach_eal_single_hyphen_arg -#undef _ -#define _(a,b) \ - else if (unformat(input, #a " %s", &s)) \ - { \ - tmp = format (0, "-%s%c", #b, 0); \ - vec_add1 (conf->eal_init_args, tmp); \ - vec_add1 (s, 0); \ - vec_add1 (conf->eal_init_args, s); \ - conf->a##_set_manually = 1; \ - } - foreach_eal_single_hyphen_mandatory_arg #undef _ else if (unformat (input, "default")) ; @@ -1651,38 +1635,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) if (error) return error; - /* I'll bet that -c and -n must be the first and second args... */ - if (!conf->coremask_set_manually) - { - vlib_thread_registration_t *tr; - uword *coremask = 0; - int i; - - /* main thread core */ - coremask = clib_bitmap_set (coremask, tm->main_lcore, 1); - - for (i = 0; i < vec_len (tm->registrations); i++) - { - tr = tm->registrations[i]; - coremask = clib_bitmap_or (coremask, tr->coremask); - } - - vec_insert (conf->eal_init_args, 2, 1); - conf->eal_init_args[1] = (u8 *) "-c"; - tmp = format (0, "%U%c", format_bitmap_hex, coremask, 0); - conf->eal_init_args[2] = tmp; - clib_bitmap_free (coremask); - } - - if (!conf->nchannels_set_manually) - { - vec_insert (conf->eal_init_args, 2, 3); - conf->eal_init_args[3] = (u8 *) "-n"; - tmp = format (0, "%d", conf->nchannels); - vec_terminate_c_string (tmp); - conf->eal_init_args[4] = tmp; - } - if (no_pci == 0 && geteuid () == 0) dpdk_bind_devices_to_uio (conf); @@ -1693,7 +1645,6 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) if (devconf->x == 0 && conf->default_devconf.x > 0) \ devconf->x = conf->default_devconf.x ; - /* *INDENT-OFF* */ pool_foreach (devconf, conf->dev_confs) { /* default per-device config items */ @@ -1737,17 +1688,9 @@ dpdk_config (vlib_main_t * vm, unformat_input_t * input) vec_add1 (conf->eal_init_args, tmp); } } - /* *INDENT-ON* */ #undef _ - /* set master-lcore */ - tmp = format (0, "--main-lcore%c", 0); - vec_add1 (conf->eal_init_args, tmp); - tmp = format (0, "%u%c", tm->main_lcore, 0); - vec_add1 (conf->eal_init_args, tmp); - - if (socket_mem) clib_warning ("socket-mem argument is deprecated"); @@ -1999,7 +1942,6 @@ dpdk_init (vlib_main_t * vm) dm->vnet_main = vnet_get_main (); dm->conf = &dpdk_config_main; - dm->conf->nchannels = 4; vec_add1 (dm->conf->eal_init_args, (u8 *) "vnet"); /* Default vlib_buffer_t flags, DISABLES tcp/udp checksumming... */ @@ -2020,10 +1962,13 @@ dpdk_init (vlib_main_t * vm) VLIB_INIT_FUNCTION (dpdk_init); -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ +static clib_error_t * +dpdk_worker_thread_init (vlib_main_t *vm) +{ + if (rte_thread_register () < 0) + clib_panic ("dpdk: cannot register thread %u - %s", vm->thread_index, + rte_strerror (rte_errno)); + return 0; +} + +VLIB_WORKER_INIT_FUNCTION (dpdk_worker_thread_init); diff --git a/src/plugins/dpdk/thread.c b/src/plugins/dpdk/thread.c deleted file mode 100644 index 3a3fcc6cea6..00000000000 --- a/src/plugins/dpdk/thread.c +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright (c) 2017 Cisco and/or its affiliates. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at: - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -static clib_error_t * -dpdk_launch_thread (void *fp, vlib_worker_thread_t * w, unsigned lcore_id) -{ - int r; - r = rte_eal_remote_launch (fp, (void *) w, lcore_id); - if (r) - return clib_error_return (0, "Failed to launch thread %u", lcore_id); - return 0; -} - -static clib_error_t * -dpdk_thread_set_lcore (u32 thread, u16 lcore) -{ - return 0; -} - -static vlib_thread_callbacks_t callbacks = { - .vlib_launch_thread_cb = &dpdk_launch_thread, - .vlib_thread_set_lcore_cb = &dpdk_thread_set_lcore, -}; - -static clib_error_t * -dpdk_thread_init (vlib_main_t * vm) -{ - vlib_thread_cb_register (vm, &callbacks); - return 0; -} - -VLIB_INIT_FUNCTION (dpdk_thread_init); - -/** @endcond */ -/* - * fd.io coding-style-patch-verification: ON - * - * Local Variables: - * eval: (c-set-style "gnu") - * End: - */ diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 760aa8b5776..f4d95c45059 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -175,6 +175,7 @@ vlib_thread_init (vlib_main_t * vm) vlib_thread_main_t *tm = &vlib_thread_main; vlib_worker_thread_t *w; vlib_thread_registration_t *tr; + cpu_set_t cpuset; u32 n_vlib_mains = 1; u32 first_index = 1; u32 i; @@ -223,17 +224,9 @@ vlib_thread_init (vlib_main_t * vm) tm->cpu_socket_bitmap = clib_bitmap_set (0, 0, 1); /* pin main thread to main_lcore */ - if (tm->cb.vlib_thread_set_lcore_cb) - { - tm->cb.vlib_thread_set_lcore_cb (0, tm->main_lcore); - } - else - { - cpu_set_t cpuset; - CPU_ZERO (&cpuset); - CPU_SET (tm->main_lcore, &cpuset); - pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); - } + CPU_ZERO (&cpuset); + CPU_SET (tm->main_lcore, &cpuset); + pthread_setaffinity_np (pthread_self (), sizeof (cpu_set_t), &cpuset); /* Set up thread 0 */ vec_validate_aligned (vlib_worker_threads, 0, CLIB_CACHE_LINE_BYTES); @@ -463,6 +456,8 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) { clib_mem_main_t *mm = &clib_mem_main; vlib_thread_main_t *tm = &vlib_thread_main; + pthread_t worker; + cpu_set_t cpuset; void *(*fp_arg) (void *) = fp; void *numa_heap; @@ -489,12 +484,6 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) } } - if (tm->cb.vlib_launch_thread_cb && !w->registration->use_pthreads) - return tm->cb.vlib_launch_thread_cb (fp, (void *) w, cpu_id); - else - { - pthread_t worker; - cpu_set_t cpuset; CPU_ZERO (&cpuset); CPU_SET (cpu_id, &cpuset); @@ -505,7 +494,6 @@ vlib_launch_thread_int (void *fp, vlib_worker_thread_t * w, unsigned cpu_id) return clib_error_return_unix (0, "pthread_setaffinity_np"); return 0; - } } static clib_error_t * @@ -1520,7 +1508,6 @@ vlib_worker_thread_fn (void *arg) { vlib_global_main_t *vgm = vlib_get_global_main (); vlib_worker_thread_t *w = (vlib_worker_thread_t *) arg; - vlib_thread_main_t *tm = vlib_get_thread_main (); vlib_main_t *vm = vlib_get_main (); clib_error_t *e; @@ -1540,10 +1527,6 @@ vlib_worker_thread_fn (void *arg) if (e) clib_error_report (e); - /* Wait until the dpdk init sequence is complete */ - while (tm->extern_thread_mgmt && tm->worker_thread_release == 0) - vlib_worker_thread_barrier_check (); - vlib_worker_loop (vm); } @@ -1586,19 +1569,6 @@ vlib_frame_queue_main_init (u32 node_index, u32 frame_queue_nelts) return (fqm - tm->frame_queue_mains); } -int -vlib_thread_cb_register (struct vlib_main_t *vm, vlib_thread_callbacks_t * cb) -{ - vlib_thread_main_t *tm = vlib_get_thread_main (); - - if (tm->extern_thread_mgmt) - return -1; - - tm->cb.vlib_launch_thread_cb = cb->vlib_launch_thread_cb; - tm->extern_thread_mgmt = 1; - return 0; -} - void vlib_process_signal_event_mt_helper (vlib_process_signal_event_mt_args_t * args) diff --git a/src/vlib/threads.h b/src/vlib/threads.h index 91727bacc23..e406dde5b07 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -233,13 +233,6 @@ typedef enum SCHED_POLICY_N, } sched_policy_t; -typedef struct -{ - clib_error_t *(*vlib_launch_thread_cb) (void *fp, vlib_worker_thread_t * w, - unsigned cpu_id); - clib_error_t *(*vlib_thread_set_lcore_cb) (u32 thread, u16 cpu); -} vlib_thread_callbacks_t; - typedef struct { /* Link list of registrations, built by constructors */ @@ -252,10 +245,6 @@ typedef struct vlib_worker_thread_t *worker_threads; - /* - * Launch all threads as pthreads, - * not eal_rte_launch (strict affinity) threads - */ int use_pthreads; /* Number of vlib_main / vnet_main clones */ @@ -297,10 +286,6 @@ typedef struct /* scheduling policy priority */ u32 sched_priority; - /* callbacks */ - vlib_thread_callbacks_t cb; - int extern_thread_mgmt; - /* NUMA-bound heap size */ uword numa_heap_size; @@ -490,8 +475,6 @@ vlib_thread_is_main_w_barrier (void) } u8 *vlib_thread_stack_init (uword thread_index); -int vlib_thread_cb_register (struct vlib_main_t *vm, - vlib_thread_callbacks_t * cb); extern void *rpc_call_main_thread_cb_fn; void -- 2.16.6