vlib: timing wheel improvements

Type: improvement
Change-Id: I9b57ed8632e0afea9f4b67972619f411c27f35f2
Signed-off-by: Damjan Marion <[email protected]>

diff --git a/src/vlib/CMakeLists.txt b/src/vlib/CMakeLists.txt
index 1f1a139..edada1b 100644 (file)
--- a/src/vlib/CMakeLists.txt
+++ b/src/vlib/CMakeLists.txt
@@ -162,6 +162,7 @@ add_vpp_library(vlib
   time.h
   trace_funcs.h
   trace.h
+  tw_funcs.h
   unix/mc_socket.h
   unix/plugin.h
   unix/unix.h

diff --git a/src/vlib/main.c b/src/vlib/main.c
index 9a983bb..6391e28 100644 (file)
--- a/src/vlib/main.c
+++ b/src/vlib/main.c
@@ -1257,15 +1257,13 @@ vlib_process_resume (vlib_main_t * vm, vlib_process_t * p)
 static void
 process_timer_start (vlib_main_t *vm, vlib_process_t *p, u32 runtime_index)
 {
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
   vlib_tw_event_t e = { .type = VLIB_TW_EVENT_T_PROCESS_NODE,
                        .index = runtime_index };
 
   if (p->resume_clock_interval == 0)
     return;
 
-  p->stop_timer_handle = TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */,
-                                             p->resume_clock_interval);
+  p->stop_timer_handle = vlib_tw_timer_start (vm, e, p->resume_clock_interval);
 }
 
 static u64
@@ -1444,14 +1442,16 @@ dispatch_suspended_process (vlib_main_t *vm, vlib_process_restore_t *r,
   return t;
 }
 
-static void
-process_expired_timer_cb (u32 *expired_timer_handles)
+static __clib_warn_unused_result u32 *
+process_expired_timers (u32 *v)
 {
   vlib_main_t *vm = vlib_get_main ();
   vlib_node_main_t *nm = &vm->node_main;
   u32 *handle;
 
-  vec_foreach (handle, expired_timer_handles)
+  v = vlib_tw_timer_expire_timers (vm, v);
+
+  vec_foreach (handle, v)
     {
       vlib_tw_event_t e = { .as_u32 = *handle };
       vlib_process_restore_t restore = {};
@@ -1477,26 +1477,7 @@ process_expired_timer_cb (u32 *expired_timer_handles)
       else
        ASSERT (0);
     }
-}
-
-static void
-vlib_tw_init (vlib_main_t *vm)
-{
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
-  tw = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
-                              CLIB_CACHE_LINE_BYTES);
-  /* Create the process timing wheel */
-  TW (tw_timer_wheel_init)
-  (tw, process_expired_timer_cb /* callback */, 1 / VLIB_TW_TICKS_PER_SECOND,
-   ~0 /* max expirations per call */);
-  vm->timing_wheel = tw;
-}
-
-static void
-vlib_tw_expire_timers (vlib_main_t *vm)
-{
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
-  TW (tw_timer_expire_timers) (tw, vlib_time_now (vm));
+  return v;
 }
 
 static_always_inline void
@@ -1509,6 +1490,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
   f64 now;
   vlib_frame_queue_main_t *fqm;
   u32 frame_queue_check_counter = 0;
+  u32 *expired_timers = 0;
 
   /* Initialize pending node vector. */
   if (is_main)
@@ -1680,7 +1662,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
          if (PREDICT_FALSE (vm->elog_trace_graph_dispatch))
            ed = ELOG_DATA (&vlib_global_main.elog_main, es);
 
-         vlib_tw_expire_timers (vm);
+         expired_timers = process_expired_timers (expired_timers);
 
          ASSERT (nm->process_restore_current != 0);
 
@@ -1739,7 +1721,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main)
            }
        }
       else
-       vlib_tw_expire_timers (vm);
+       expired_timers = process_expired_timers (expired_timers);
 
       vlib_increment_main_loop_counter (vm);
       /* Record time stamp in case there are no enabled nodes and above

diff --git a/src/vlib/main.h b/src/vlib/main.h
index 907346b..2a4ec83 100644 (file)
--- a/src/vlib/main.h
+++ b/src/vlib/main.h
@@ -260,6 +260,7 @@ typedef struct vlib_main_t
 
   /* Timing wheel for scheduling time-based node dispatch. */
   void *timing_wheel;
+  u32 n_tw_timers;
 
 #ifdef CLIB_SANITIZE_ADDR
   /* address sanitizer stack save */

diff --git a/src/vlib/node.h b/src/vlib/node.h
index 22d5a48..690a1b0 100644 (file)
--- a/src/vlib/node.h
+++ b/src/vlib/node.h
@@ -731,23 +731,6 @@ typedef struct
 }
 vlib_signal_timed_event_data_t;
 
-typedef enum
-{
-  VLIB_TW_EVENT_T_PROCESS_NODE = 1,
-  VLIB_TW_EVENT_T_TIMED_EVENT = 2,
-  VLIB_TW_EVENT_T_SCHED_NODE = 3,
-} vlib_tw_event_type_t;
-
-typedef union
-{
-  struct
-  {
-    u32 type : 2; /* vlib_tw_event_type_t */
-    u32 index : 30;
-  };
-  u32 as_u32;
-} vlib_tw_event_t;
-
 typedef struct
 {
   clib_march_variant_type_t index;

diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h
index ef8cf78..b508828 100644 (file)
--- a/src/vlib/node_funcs.h
+++ b/src/vlib/node_funcs.h
@@ -47,11 +47,8 @@
 
 #include <vppinfra/clib.h>
 #include <vppinfra/fifo.h>
-#include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
 #include <vppinfra/interrupt.h>
 
-#define VLIB_TW_TICKS_PER_SECOND 1e5 /* 10 us */
-
 #ifdef CLIB_SANITIZE_ADDR
 #include <sanitizer/asan_interface.h>
 #endif
@@ -271,7 +268,6 @@ vlib_node_is_scheduled (vlib_main_t *vm, u32 node_index)
 always_inline void
 vlib_node_schedule (vlib_main_t *vm, u32 node_index, f64 dt)
 {
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
   u64 ticks;
 
   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index);
@@ -286,21 +282,18 @@ vlib_node_schedule (vlib_main_t *vm, u32 node_index, f64 dt)
   dt = flt_round_nearest (dt * VLIB_TW_TICKS_PER_SECOND);
   ticks = clib_max ((u64) dt, 1);
 
-  rt->stop_timer_handle_plus_1 =
-    1 + TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, ticks);
+  rt->stop_timer_handle_plus_1 = 1 + vlib_tw_timer_start (vm, e, ticks);
 }
 
 always_inline void
 vlib_node_unschedule (vlib_main_t *vm, u32 node_index)
 {
   vlib_node_runtime_t *rt = vlib_node_get_runtime (vm, node_index);
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
 
   ASSERT (vm == vlib_get_main ());
   ASSERT (vlib_node_is_scheduled (vm, node_index) == 1);
 
-  TW (tw_timer_stop) (tw, rt->stop_timer_handle_plus_1);
-
+  vlib_tw_timer_stop (vm, rt->stop_timer_handle_plus_1 - 1);
   rt->stop_timer_handle_plus_1 = 0;
 }
 
@@ -1004,7 +997,6 @@ vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm,
                                  vlib_node_t *n, vlib_process_t *p, uword t,
                                  uword n_data_elts, uword n_data_elt_bytes)
 {
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
   uword add_to_pending = 0, delete_from_wheel = 0;
   u8 *data_to_be_written_by_caller;
   vec_attr_t va = { .elt_sz = n_data_elt_bytes };
@@ -1052,7 +1044,7 @@ vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm,
       break;
     }
 
-  if (TW (tw_timer_handle_is_free) (tw, p->stop_timer_handle))
+  if (vlib_tw_timer_handle_is_free (vm, p->stop_timer_handle))
     delete_from_wheel = 0;
 
   /* Never add current process to pending vector since current process is
@@ -1071,7 +1063,7 @@ vlib_process_signal_event_helper (vlib_main_t *vm, vlib_node_main_t *nm,
 
   if (delete_from_wheel)
     {
-      TW (tw_timer_stop) (tw, p->stop_timer_handle);
+      vlib_tw_timer_stop (vm, p->stop_timer_handle);
       p->stop_timer_handle = ~0;
     }
 
@@ -1114,7 +1106,6 @@ vlib_process_signal_event_at_time (vlib_main_t * vm,
                                   uword type_opaque,
                                   uword n_data_elts, uword n_data_elt_bytes)
 {
-  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
   vlib_node_main_t *nm = &vm->node_main;
   vlib_node_t *n = vlib_get_node (vm, node_index);
   vlib_process_t *p = vec_elt (nm->processes, n->runtime_index);
@@ -1153,13 +1144,12 @@ vlib_process_signal_event_at_time (vlib_main_t * vm,
       te->event_type_index = t;
 
       p->stop_timer_handle =
-       TW (tw_timer_start) (tw,
+       vlib_tw_timer_start (vm,
                             (vlib_tw_event_t){
                               .type = VLIB_TW_EVENT_T_TIMED_EVENT,
                               .index = te - nm->signal_timed_event_data_pool,
-                            }
-                              .as_u32,
-                            0 /* timer_id */, dt * VLIB_TW_TICKS_PER_SECOND);
+                            },
+                            dt * VLIB_TW_TICKS_PER_SECOND);
 
       /* Inline data big enough to hold event? */
       if (te->n_data_bytes < sizeof (te->inline_event_data))

diff --git a/src/vlib/time.h b/src/vlib/time.h
index a9dc539..fa8cdb2 100644 (file)
--- a/src/vlib/time.h
+++ b/src/vlib/time.h
@@ -7,13 +7,13 @@
 #define included_vlib_time_h
 
 #include <vlib/vlib.h>
-#include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
+#include <vlib/tw_funcs.h>
 
 static inline f64
 vlib_time_get_next_timer (vlib_main_t *vm)
 {
   TWT (tw_timer_wheel) *wheel = vm->timing_wheel;
-  return TW (tw_timer_first_expires_in_ticks) (wheel) * wheel->timer_interval;
+  return vlib_tw_timer_first_expires_in_ticks (vm) * wheel->timer_interval;
 }
 
 static inline void

diff --git a/src/vlib/tw_funcs.h b/src/vlib/tw_funcs.h
new file mode 100644 (file)
index 0000000..c64aaa3
--- /dev/null
+++ b/src/vlib/tw_funcs.h
@@ -0,0 +1,90 @@
+/* SPDX-License-Identifier: Apache-2.0
+ * Copyright (c) 2025 Cisco Systems, Inc.
+ */
+
+#ifndef __vlib_tw_funcs_h__
+#define __vlib_tw_funcs_h__
+
+#include <vppinfra/tw_timer_1t_3w_1024sl_ov.h>
+#define VLIB_TW_TICKS_PER_SECOND 1e5 /* 10 us */
+
+typedef enum
+{
+  VLIB_TW_EVENT_T_PROCESS_NODE = 1,
+  VLIB_TW_EVENT_T_TIMED_EVENT = 2,
+  VLIB_TW_EVENT_T_SCHED_NODE = 3,
+} vlib_tw_event_type_t;
+
+typedef union
+{
+  struct
+  {
+    u32 type : 2; /* vlib_tw_event_type_t */
+    u32 index : 30;
+  };
+  u32 as_u32;
+} vlib_tw_event_t;
+
+static_always_inline u32
+vlib_tw_timer_start (vlib_main_t *vm, vlib_tw_event_t e, u64 interval)
+{
+  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
+  vm->n_tw_timers++;
+  return TW (tw_timer_start) (tw, e.as_u32, 0 /* timer_id */, interval);
+}
+
+static_always_inline void
+vlib_tw_timer_stop (vlib_main_t *vm, u32 handle)
+{
+  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
+  ASSERT (vm->n_tw_timers > 0);
+  vm->n_tw_timers--;
+  TW (tw_timer_stop) (tw, handle);
+}
+
+static_always_inline int
+vlib_tw_timer_handle_is_free (vlib_main_t *vm, u32 handle)
+{
+  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
+  return TW (tw_timer_handle_is_free) (tw, handle);
+}
+
+static_always_inline u32
+vlib_tw_timer_first_expires_in_ticks (vlib_main_t *vm)
+{
+  return TW (tw_timer_first_expires_in_ticks) (
+    (TWT (tw_timer_wheel) *) vm->timing_wheel);
+}
+
+static_always_inline void
+vlib_tw_init (vlib_main_t *vm)
+{
+  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
+  tw = clib_mem_alloc_aligned (sizeof (TWT (tw_timer_wheel)),
+                              CLIB_CACHE_LINE_BYTES);
+  /* Create the process timing wheel */
+  TW (tw_timer_wheel_init)
+  (tw, 0 /* callback */, 1 / VLIB_TW_TICKS_PER_SECOND,
+   ~0 /* max expirations per call */);
+  vm->timing_wheel = tw;
+  vm->n_tw_timers = 0;
+}
+
+static_always_inline u32 *
+vlib_tw_timer_expire_timers (vlib_main_t *vm, u32 *v)
+{
+  TWT (tw_timer_wheel) *tw = (TWT (tw_timer_wheel) *) vm->timing_wheel;
+
+  vec_reset_length (v);
+
+  if (vm->n_tw_timers > 0)
+    {
+      v = TW (tw_timer_expire_timers_vec) (tw, vlib_time_now (vm), v);
+      ASSERT (vec_len (v) <= vm->n_tw_timers);
+      vm->n_tw_timers -= vec_len (v);
+    }
+
+  return v;
+}
+
+#endif /* __vlib_tw_funcs_h__ */

diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c
index 302d1eb..2edc5ec 100644 (file)
--- a/src/vlib/unix/input.c
+++ b/src/vlib/unix/input.c
@@ -185,8 +185,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node,
     else if (is_main && vector_rate < 2 && vm->api_queue_nonempty == 0
             && nm->input_node_counts_by_state[VLIB_NODE_STATE_POLLING] == 0)
       {
-       ticks_until_expiration = TW (tw_timer_first_expires_in_ticks) (
-         (TWT (tw_timer_wheel) *) vm->timing_wheel);
+       ticks_until_expiration = vlib_tw_timer_first_expires_in_ticks (vm);
 
        /* Nothing on the fast wheel, sleep 10ms */
        if (ticks_until_expiration == TW_SLOTS_PER_RING)

diff --git a/src/vlib/vlib.h b/src/vlib/vlib.h
index 36f8a36..1e2b25e 100644 (file)
--- a/src/vlib/vlib.h
+++ b/src/vlib/vlib.h
@@ -71,6 +71,7 @@ typedef u32 vlib_log_class_t;
 #include <vlib/threads.h>
 #include <vlib/physmem_funcs.h>
 #include <vlib/buffer_funcs.h>
+#include <vlib/tw_funcs.h>
 #include <vlib/error_funcs.h>
 #include <vlib/format_funcs.h>
 #include <vlib/node_funcs.h>