From: Damjan Marion Date: Wed, 9 Apr 2025 16:36:18 +0000 (+0200) Subject: vlib: timing wheel improvements X-Git-Tag: v25.10-rc0~92 X-Git-Url: https://gerrit.fd.io/r/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F77%2F42777%2F10;p=vpp.git vlib: timing wheel improvements Type: improvement Change-Id: I9b57ed8632e0afea9f4b67972619f411c27f35f2 Signed-off-by: Damjan Marion --- diff --git a/src/vlib/CMakeLists.txt b/src/vlib/CMakeLists.txt index 1f1a1396d0d..edada1bc798 100644 --- a/src/vlib/CMakeLists.txt +++ b/src/vlib/CMakeLists.txt @@ -162,6 +162,7 @@ add_vpp_library(vlib time.h trace_funcs.h trace.h + tw_funcs.h unix/mc_socket.h unix/plugin.h unix/unix.h diff --git a/src/vlib/main.c b/src/vlib/main.c index 9a983bb5451..6391e287b53 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1257,15 +1257,13 @@ vlib_process_resume (vlib_main_t * vm, vlib_process_t * p) static void process_timer_start (vlib_main_t *vm, vlib_process_t *p, u32 runtime_index) { - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; vlib_tw_event_t e = { .type = VLIB_TW_EVENT_T_PROCESS_NODE, .index = runtime_index }; if (p->resume_clock_interval == 0) return; - p->stop_timer_handle = TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, - p->resume_clock_interval); + p->stop_timer_handle = vlib_tw_timer_start (vm, e, p->resume_clock_interval); } static u64 @@ -1444,14 +1442,16 @@ dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r, return t; } -static void -process_expired_timer_cb (u32 *expired_timer_handles) +static __clib_warn_unused_result u32 * +process_expired_timers (u32 *v) { vlib_main_t *vm = vlib_get_main (); vlib_node_main_t *nm = &vm->node_main; u32 *handle; - vec_foreach (handle, expired_timer_handles) + v = vlib_tw_timer_expire_timers (vm, v); + + vec_foreach (handle, v) { vlib_tw_event_t e = { .as_u32 = *handle }; vlib_process_restore_t restore = {}; @@ -1477,26 +1477,7 @@ process_expired_timer_cb (u32 *expired_timer_handles) else ASSERT (0); } -} - -static void -vlib_tw_init (vlib_main_t *vm) -{ - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; - tw = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), - CLIB_CACHE_LINE_BYTES); - /* Create the process timing wheel */ - TW (tw_timer_wheel_init) - (tw, process_expired_timer_cb /* callback */, 1 / VLIB_TW_TICKS_PER_SECOND, - ~0 /* max expirations per call */); - vm->timing_wheel = tw; -} - -static void -vlib_tw_expire_timers (vlib_main_t *vm) -{ - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; - TW (tw_timer_expire_timers) (tw, vlib_time_now (vm)); + return v; } static_always_inline void @@ -1509,6 +1490,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) f64 now; vlib_frame_queue_main_t *fqm; u32 frame_queue_check_counter = 0; + u32 *expired_timers = 0; /* Initialize pending node vector. */ if (is_main) @@ -1680,7 +1662,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) if (PREDICT_FALSE (vm->elog_trace_graph_dispatch)) ed = ELOG_DATA (&vlib_global_main.elog_main, es); - vlib_tw_expire_timers (vm); + expired_timers = process_expired_timers (expired_timers); ASSERT (nm->process_restore_current != 0); @@ -1739,7 +1721,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } } else - vlib_tw_expire_timers (vm); + expired_timers = process_expired_timers (expired_timers); vlib_increment_main_loop_counter (vm); /* Record time stamp in case there are no enabled nodes and above diff --git a/src/vlib/main.h b/src/vlib/main.h index 907346beda2..2a4ec839ec5 100644 --- a/src/vlib/main.h +++ b/src/vlib/main.h @@ -260,6 +260,7 @@ typedef struct vlib_main_t /* Timing wheel for scheduling time-based node dispatch. */ void *timing_wheel; + u32 n_tw_timers; #ifdef CLIB_SANITIZE_ADDR /* address sanitizer stack save */ diff --git a/src/vlib/node.h b/src/vlib/node.h index 22d5a4843d7..690a1b0a8b7 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -731,23 +731,6 @@ typedef struct } vlib_signal_timed_event_data_t; -typedef enum -{ - VLIB_TW_EVENT_T_PROCESS_NODE = 1, - VLIB_TW_EVENT_T_TIMED_EVENT = 2, - VLIB_TW_EVENT_T_SCHED_NODE = 3, -} vlib_tw_event_type_t; - -typedef union -{ - struct - { - u32 type : 2; /* vlib_tw_event_type_t */ - u32 index : 30; - }; - u32 as_u32; -} vlib_tw_event_t; - typedef struct { clib_march_variant_type_t index; diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index ef8cf78e838..b5088282168 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -47,11 +47,8 @@ #include #include -#include #include -#define VLIB_TW_TICKS_PER_SECOND 1e5 /* 10 us */ - #ifdef CLIB_SANITIZE_ADDR #include #endif @@ -271,7 +268,6 @@ vlib_node_is_scheduled (vlib_main_t *vm, u32 node_index) always_inline void vlib_node_schedule (vlib_main_t *vm, u32 node_index, f64 dt) { - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; u64 ticks; vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index); @@ -286,21 +282,18 @@ vlib_node_schedule (vlib_main_t *vm, u32 node_index, f64 dt) dt = flt_round_nearest (dt * VLIB_TW_TICKS_PER_SECOND); ticks = clib_max ((u64) dt, 1); - rt->stop_timer_handle_plus_1 = - 1 + TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, ticks); + rt->stop_timer_handle_plus_1 = 1 + vlib_tw_timer_start (vm, e, ticks); } always_inline void vlib_node_unschedule (vlib_main_t *vm, u32 node_index) { vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index); - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; ASSERT (vm == vlib_get_main ()); ASSERT (vlib_node_is_scheduled (vm, node_index) == 1); - TW (tw_timer_stop) (tw, rt->stop_timer_handle_plus_1); - + vlib_tw_timer_stop (vm, rt->stop_timer_handle_plus_1 - 1); rt->stop_timer_handle_plus_1 = 0; } @@ -1004,7 +997,6 @@ vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm, vlib_node_t *n, vlib_process_t *p, uword t, uword n_data_elts, uword n_data_elt_bytes) { - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; uword add_to_pending = 0, delete_from_wheel = 0; u8 *data_to_be_written_by_caller; vec_attr_t va = { .elt_sz = n_data_elt_bytes }; @@ -1052,7 +1044,7 @@ vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm, break; } - if (TW (tw_timer_handle_is_free) (tw, p->stop_timer_handle)) + if (vlib_tw_timer_handle_is_free (vm, p->stop_timer_handle)) delete_from_wheel = 0; /* Never add current process to pending vector since current process is @@ -1071,7 +1063,7 @@ vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm, if (delete_from_wheel) { - TW (tw_timer_stop) (tw, p->stop_timer_handle); + vlib_tw_timer_stop (vm, p->stop_timer_handle); p->stop_timer_handle = ~0; } @@ -1114,7 +1106,6 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, uword type_opaque, uword n_data_elts, uword n_data_elt_bytes) { - TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vlib_get_node (vm, node_index); vlib_process_t *p = vec_elt (nm->processes, n->runtime_index); @@ -1153,13 +1144,12 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, te->event_type_index = t; p->stop_timer_handle = - TW (tw_timer_start) (tw, + vlib_tw_timer_start (vm, (vlib_tw_event_t){ .type = VLIB_TW_EVENT_T_TIMED_EVENT, .index = te - nm->signal_timed_event_data_pool, - } - .as_u32, - 0 /* timer_id */, dt * VLIB_TW_TICKS_PER_SECOND); + }, + dt * VLIB_TW_TICKS_PER_SECOND); /* Inline data big enough to hold event? */ if (te->n_data_bytes < sizeof (te->inline_event_data)) diff --git a/src/vlib/time.h b/src/vlib/time.h index a9dc5395661..fa8cdb2ec8d 100644 --- a/src/vlib/time.h +++ b/src/vlib/time.h @@ -7,13 +7,13 @@ #define included_vlib_time_h #include -#include +#include static inline f64 vlib_time_get_next_timer (vlib_main_t *vm) { TWT (tw_timer_wheel) *wheel = vm->timing_wheel; - return TW (tw_timer_first_expires_in_ticks) (wheel) * wheel->timer_interval; + return vlib_tw_timer_first_expires_in_ticks (vm) * wheel->timer_interval; } static inline void diff --git a/src/vlib/tw_funcs.h b/src/vlib/tw_funcs.h new file mode 100644 index 00000000000..c64aaa3db45 --- /dev/null +++ b/src/vlib/tw_funcs.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: Apache-2.0 + * Copyright (c) 2025 Cisco Systems, Inc. + */ + +#ifndef __vlib_tw_funcs_h__ +#define __vlib_tw_funcs_h__ + +#include +#define VLIB_TW_TICKS_PER_SECOND 1e5 /* 10 us */ + +typedef enum +{ + VLIB_TW_EVENT_T_PROCESS_NODE = 1, + VLIB_TW_EVENT_T_TIMED_EVENT = 2, + VLIB_TW_EVENT_T_SCHED_NODE = 3, +} vlib_tw_event_type_t; + +typedef union +{ + struct + { + u32 type : 2; /* vlib_tw_event_type_t */ + u32 index : 30; + }; + u32 as_u32; +} vlib_tw_event_t; + +static_always_inline u32 +vlib_tw_timer_start (vlib_main_t *vm, vlib_tw_event_t e, u64 interval) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + vm->n_tw_timers++; + return TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, interval); +} + +static_always_inline void +vlib_tw_timer_stop (vlib_main_t *vm, u32 handle) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + ASSERT (vm->n_tw_timers > 0); + vm->n_tw_timers--; + TW (tw_timer_stop) (tw, handle); +} + +static_always_inline int +vlib_tw_timer_handle_is_free (vlib_main_t *vm, u32 handle) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + return TW (tw_timer_handle_is_free) (tw, handle); +} + +static_always_inline u32 +vlib_tw_timer_first_expires_in_ticks (vlib_main_t *vm) +{ + return TW (tw_timer_first_expires_in_ticks) ( + (TWT (tw_timer_wheel) *) vm->timing_wheel); +} + +static_always_inline void +vlib_tw_init (vlib_main_t *vm) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + tw = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)), + CLIB_CACHE_LINE_BYTES); + /* Create the process timing wheel */ + TW (tw_timer_wheel_init) + (tw, 0 /* callback */, 1 / VLIB_TW_TICKS_PER_SECOND, + ~0 /* max expirations per call */); + vm->timing_wheel = tw; + vm->n_tw_timers = 0; +} + +static_always_inline u32 * +vlib_tw_timer_expire_timers (vlib_main_t *vm, u32 *v) +{ + TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel; + + vec_reset_length (v); + + if (vm->n_tw_timers > 0) + { + v = TW (tw_timer_expire_timers_vec) (tw, vlib_time_now (vm), v); + ASSERT (vec_len (v) <= vm->n_tw_timers); + vm->n_tw_timers -= vec_len (v); + } + + return v; +} + +#endif /* __vlib_tw_funcs_h__ */ diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c index 302d1eb1194..2edc5ecb760 100644 --- a/src/vlib/unix/input.c +++ b/src/vlib/unix/input.c @@ -185,8 +185,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, else if (is_main && vector_rate < 2 && vm->api_queue_nonempty == 0 && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0) { - ticks_until_expiration = TW (tw_timer_first_expires_in_ticks) ( - (TWT (tw_timer_wheel) *) vm->timing_wheel); + ticks_until_expiration = vlib_tw_timer_first_expires_in_ticks (vm); /* Nothing on the fast wheel, sleep 10ms */ if (ticks_until_expiration == TW_SLOTS_PER_RING) diff --git a/src/vlib/vlib.h b/src/vlib/vlib.h index 36f8a361abc..1e2b25eba3b 100644 --- a/src/vlib/vlib.h +++ b/src/vlib/vlib.h @@ -71,6 +71,7 @@ typedef u32 vlib_log_class_t; #include #include #include +#include #include #include #include