From cc8249c5fde1b00b043066617d35325dd1606cd6 Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Sun, 23 Jul 2023 14:24:22 +0200 Subject: [PATCH] vlib: interrupt mode support for pre-input nodes Type: improvement Change-Id: Ic6e60597d2be63e3a0ae4399a81dbbd72392f30d Signed-off-by: Damjan Marion --- src/vlib/main.c | 34 +++++++++++++++++++++++++++------- src/vlib/node.c | 5 ++++- src/vlib/node.h | 5 +++-- src/vlib/node_funcs.h | 14 ++++++++++---- src/vlib/threads.c | 10 ++++++++-- src/vlib/unix/input.c | 2 +- 6 files changed, 53 insertions(+), 17 deletions(-) diff --git a/src/vlib/main.c b/src/vlib/main.c index 0f67bb86554..9d34aa1d948 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1471,8 +1471,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) else cpu_time_now = clib_cpu_time_now (); - /* Pre-allocate interupt runtime indices and lock. */ - vec_validate_aligned (nm->pending_interrupts, 0, CLIB_CACHE_LINE_BYTES); + nm->pending_interrupts = 0; /* Pre-allocate expired nodes. */ if (!nm->polling_threshold_vector_length) @@ -1505,6 +1504,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) while (1) { vlib_node_runtime_t *n; + u8 pending_interrupts; if (PREDICT_FALSE (_vec_len (vm->pending_rpc_requests) > 0)) { @@ -1552,6 +1552,27 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) /* frame */ 0, cpu_time_now); + pending_interrupts = + __atomic_load_n (&nm->pending_interrupts, __ATOMIC_ACQUIRE); + + if (pending_interrupts) + { + int int_num = -1; + nm->pending_interrupts = 0; + + while ((int_num = clib_interrupt_get_next ( + nm->pre_input_node_interrupts, int_num)) != -1) + { + vlib_node_runtime_t *n; + clib_interrupt_clear (nm->pre_input_node_interrupts, int_num); + n = vec_elt_at_index ( + nm->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT], int_num); + cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_PRE_INPUT, + VLIB_NODE_STATE_INTERRUPT, + /* frame */ 0, cpu_time_now); + } + } + /* Next process input nodes. */ vec_foreach (n, nm->nodes_by_type[VLIB_NODE_TYPE_INPUT]) cpu_time_now = dispatch_node (vm, n, @@ -1563,16 +1584,15 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) if (PREDICT_TRUE (is_main && vm->queue_signal_pending == 0)) vm->queue_signal_callback (vm); - if (__atomic_load_n (nm->pending_interrupts, __ATOMIC_ACQUIRE)) + if (pending_interrupts) { int int_num = -1; - *nm->pending_interrupts = 0; - while ((int_num = - clib_interrupt_get_next (nm->interrupts, int_num)) != -1) + while ((int_num = clib_interrupt_get_next (nm->input_node_interrupts, + int_num)) != -1) { vlib_node_runtime_t *n; - clib_interrupt_clear (nm->interrupts, int_num); + clib_interrupt_clear (nm->input_node_interrupts, int_num); n = vec_elt_at_index (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], int_num); cpu_time_now = dispatch_node (vm, n, VLIB_NODE_TYPE_INPUT, diff --git a/src/vlib/node.c b/src/vlib/node.c index 4d1ef989e7d..8b726ff9aad 100644 --- a/src/vlib/node.c +++ b/src/vlib/node.c @@ -530,7 +530,10 @@ vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, char *fmt, vec_add2_aligned (nm->nodes_by_type[n->type], rt, 1, /* align */ CLIB_CACHE_LINE_BYTES); if (n->type == VLIB_NODE_TYPE_INPUT) - clib_interrupt_resize (&nm->interrupts, + clib_interrupt_resize (&nm->input_node_interrupts, + vec_len (nm->nodes_by_type[n->type])); + else if (n->type == VLIB_NODE_TYPE_PRE_INPUT) + clib_interrupt_resize (&nm->pre_input_node_interrupts, vec_len (nm->nodes_by_type[n->type])); n->runtime_index = rt - nm->nodes_by_type[n->type]; } diff --git a/src/vlib/node.h b/src/vlib/node.h index e40208b4bed..955ffb98600 100644 --- a/src/vlib/node.h +++ b/src/vlib/node.h @@ -690,8 +690,9 @@ typedef struct vlib_node_runtime_t *nodes_by_type[VLIB_N_NODE_TYPE]; /* Node runtime indices for input nodes with pending interrupts. */ - void *interrupts; - volatile u32 *pending_interrupts; + void *input_node_interrupts; + void *pre_input_node_interrupts; + volatile u8 pending_interrupts; /* Input nodes are switched from/to interrupt to/from polling mode when average vector length goes above/below polling/interrupt diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index a1fca14faad..66f079c0547 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -252,15 +252,21 @@ vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index) { vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vec_elt (nm->nodes, node_index); + void *interrupts; - ASSERT (n->type == VLIB_NODE_TYPE_INPUT); + if (n->type == VLIB_NODE_TYPE_INPUT) + interrupts = nm->input_node_interrupts; + else if (n->type == VLIB_NODE_TYPE_PRE_INPUT) + interrupts = nm->pre_input_node_interrupts; + else + ASSERT (0); if (vm != vlib_get_main ()) - clib_interrupt_set_atomic (nm->interrupts, n->runtime_index); + clib_interrupt_set_atomic (interrupts, n->runtime_index); else - clib_interrupt_set (nm->interrupts, n->runtime_index); + clib_interrupt_set (interrupts, n->runtime_index); - __atomic_store_n (nm->pending_interrupts, 1, __ATOMIC_RELEASE); + __atomic_store_n (&nm->pending_interrupts, 1, __ATOMIC_RELEASE); } always_inline vlib_process_t * diff --git a/src/vlib/threads.c b/src/vlib/threads.c index adf225ba87f..4b70642d3cb 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -699,8 +699,11 @@ start_workers (vlib_main_t * vm) vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], CLIB_CACHE_LINE_BYTES); clib_interrupt_init ( - &nm_clone->interrupts, + &nm_clone->input_node_interrupts, vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); + clib_interrupt_init ( + &nm_clone->pre_input_node_interrupts, + vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { vlib_node_t *n = vlib_get_node (vm, rt->node_index); @@ -1022,8 +1025,11 @@ vlib_worker_thread_node_refork (void) vec_dup_aligned (nm->nodes_by_type[VLIB_NODE_TYPE_INPUT], CLIB_CACHE_LINE_BYTES); clib_interrupt_resize ( - &nm_clone->interrupts, + &nm_clone->input_node_interrupts, vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT])); + clib_interrupt_resize ( + &nm_clone->pre_input_node_interrupts, + vec_len (nm_clone->nodes_by_type[VLIB_NODE_TYPE_PRE_INPUT])); vec_foreach (rt, nm_clone->nodes_by_type[VLIB_NODE_TYPE_INPUT]) { diff --git a/src/vlib/unix/input.c b/src/vlib/unix/input.c index 9c7c54f6b1b..39c124cf8df 100644 --- a/src/vlib/unix/input.c +++ b/src/vlib/unix/input.c @@ -250,7 +250,7 @@ linux_epoll_input_inline (vlib_main_t * vm, vlib_node_runtime_t * node, while (nanosleep (&ts, &tsrem) < 0) ts = tsrem; if (*vlib_worker_threads->wait_at_barrier || - *nm->pending_interrupts) + nm->pending_interrupts) goto done; } } -- 2.16.6