From eee099e9579083fbce665e8c4a3617b9f0e5ea2f Mon Sep 17 00:00:00 2001 From: Damjan Marion Date: Sat, 1 May 2021 14:56:13 +0200 Subject: [PATCH] vlib: multiarch vlib_frame_queue_dequeue() Fact that it needs to copy buffer indices justifies this move. Type: improvement Change-Id: I5eb815ccc4cca0ef70b092eb83a49b713efdcbeb Signed-off-by: Damjan Marion --- src/vlib/buffer_funcs.c | 123 ++++++++++++++++++++++++++++++++++++++++++ src/vlib/buffer_funcs.h | 5 ++ src/vlib/main.c | 4 +- src/vlib/threads.c | 139 ------------------------------------------------ src/vlib/threads.h | 3 -- 5 files changed, 131 insertions(+), 143 deletions(-) diff --git a/src/vlib/buffer_funcs.c b/src/vlib/buffer_funcs.c index a0edd7ec60b..fcef2d80f35 100644 --- a/src/vlib/buffer_funcs.c +++ b/src/vlib/buffer_funcs.c @@ -361,6 +361,127 @@ CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_thread_fn) CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_thread_fn); +/* + * Check the frame queue to see if any frames are available. + * If so, pull the packets off the frames and put them to + * the handoff node. + */ +u32 __clib_section (".vlib_frame_queue_dequeue_fn") +CLIB_MULTIARCH_FN (vlib_frame_queue_dequeue_fn) +(vlib_main_t *vm, vlib_frame_queue_main_t *fqm) +{ + u32 thread_id = vm->thread_index; + vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id]; + vlib_frame_queue_elt_t *elt; + u32 *from, *to; + vlib_frame_t *f; + int msg_type; + int processed = 0; + u32 vectors = 0; + + ASSERT (fq); + ASSERT (vm == vlib_global_main.vlib_mains[thread_id]); + + if (PREDICT_FALSE (fqm->node_index == ~0)) + return 0; + /* + * Gather trace data for frame queues + */ + if (PREDICT_FALSE (fq->trace)) + { + frame_queue_trace_t *fqt; + frame_queue_nelt_counter_t *fqh; + u32 elix; + + fqt = &fqm->frame_queue_traces[thread_id]; + + fqt->nelts = fq->nelts; + fqt->head = fq->head; + fqt->head_hint = fq->head_hint; + fqt->tail = fq->tail; + fqt->threshold = fq->vector_threshold; + fqt->n_in_use = fqt->tail - fqt->head; + if (fqt->n_in_use >= fqt->nelts) + { + // if beyond max then use max + fqt->n_in_use = fqt->nelts - 1; + } + + /* Record the number of elements in use in the histogram */ + fqh = &fqm->frame_queue_histogram[thread_id]; + fqh->count[fqt->n_in_use]++; + + /* Record a snapshot of the elements in use */ + for (elix = 0; elix < fqt->nelts; elix++) + { + elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1)); + if (1 || elt->valid) + { + fqt->n_vectors[elix] = elt->n_vectors; + } + } + fqt->written = 1; + } + + while (1) + { + vlib_buffer_t *b; + if (fq->head == fq->tail) + { + fq->head_hint = fq->head; + return processed; + } + + elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1)); + + if (!elt->valid) + { + fq->head_hint = fq->head; + return processed; + } + + from = elt->buffer_index; + msg_type = elt->msg_type; + + ASSERT (msg_type == VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME); + ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE); + + f = vlib_get_frame_to_node (vm, fqm->node_index); + + /* If the first vector is traced, set the frame trace flag */ + b = vlib_get_buffer (vm, from[0]); + if (b->flags & VLIB_BUFFER_IS_TRACED) + f->frame_flags |= VLIB_NODE_FLAG_TRACE; + + to = vlib_frame_vector_args (f); + + vlib_buffer_copy_indices (to, from, elt->n_vectors); + + vectors += elt->n_vectors; + f->n_vectors = elt->n_vectors; + vlib_put_frame_to_node (vm, fqm->node_index, f); + + elt->valid = 0; + elt->n_vectors = 0; + elt->msg_type = 0xfefefefe; + CLIB_MEMORY_BARRIER (); + fq->head++; + processed++; + + /* + * Limit the number of packets pushed into the graph + */ + if (vectors >= fq->vector_threshold) + { + fq->head_hint = fq->head; + return processed; + } + } + ASSERT (0); + return processed; +} +CLIB_MARCH_FN_REGISTRATION (vlib_frame_queue_dequeue_fn); + #ifndef CLIB_MARCH_VARIANT vlib_buffer_func_main_t vlib_buffer_func_main; @@ -374,6 +495,8 @@ vlib_buffer_funcs_init (vlib_main_t *vm) CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_single_next_fn); bfm->buffer_enqueue_to_thread_fn = CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_thread_fn); + bfm->frame_queue_dequeue_fn = + CLIB_MARCH_FN_POINTER (vlib_frame_queue_dequeue_fn); return 0; } diff --git a/src/vlib/buffer_funcs.h b/src/vlib/buffer_funcs.h index 7829986d643..d579e1bc2e1 100644 --- a/src/vlib/buffer_funcs.h +++ b/src/vlib/buffer_funcs.h @@ -62,11 +62,16 @@ typedef void (vlib_buffer_enqueue_to_single_next_fn_t) ( typedef u32 (vlib_buffer_enqueue_to_thread_fn_t) ( vlib_main_t *vm, u32 frame_queue_index, u32 *buffer_indices, u16 *thread_indices, u32 n_packets, int drop_on_congestion); + +typedef u32 (vlib_frame_queue_dequeue_fn_t) (vlib_main_t *vm, + vlib_frame_queue_main_t *fqm); + typedef struct { vlib_buffer_enqueue_to_next_fn_t *buffer_enqueue_to_next_fn; vlib_buffer_enqueue_to_single_next_fn_t *buffer_enqueue_to_single_next_fn; vlib_buffer_enqueue_to_thread_fn_t *buffer_enqueue_to_thread_fn; + vlib_frame_queue_dequeue_fn_t *frame_queue_dequeue_fn; } vlib_buffer_func_main_t; extern vlib_buffer_func_main_t vlib_buffer_func_main; diff --git a/src/vlib/main.c b/src/vlib/main.c index f833aa25acf..c7c4aba3080 100644 --- a/src/vlib/main.c +++ b/src/vlib/main.c @@ -1581,6 +1581,8 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) if (PREDICT_FALSE (vm->check_frame_queues + frame_queue_check_counter)) { u32 processed = 0; + vlib_frame_queue_dequeue_fn_t *fn = + vlib_buffer_func_main.frame_queue_dequeue_fn; if (vm->check_frame_queues) { @@ -1589,7 +1591,7 @@ vlib_main_or_worker_loop (vlib_main_t * vm, int is_main) } vec_foreach (fqm, tm->frame_queue_mains) - processed += vlib_frame_queue_dequeue (vm, fqm); + processed += (fn) (vm, fqm); /* No handoff queue work found? */ if (processed) diff --git a/src/vlib/threads.c b/src/vlib/threads.c index 11d5a7220ad..c65460e3234 100644 --- a/src/vlib/threads.c +++ b/src/vlib/threads.c @@ -1644,145 +1644,6 @@ vlib_worker_wait_one_loop (void) return; } -/* - * Check the frame queue to see if any frames are available. - * If so, pull the packets off the frames and put them to - * the handoff node. - */ -int -vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm) -{ - u32 thread_id = vm->thread_index; - vlib_frame_queue_t *fq = fqm->vlib_frame_queues[thread_id]; - vlib_frame_queue_elt_t *elt; - u32 *from, *to; - vlib_frame_t *f; - int msg_type; - int processed = 0; - u32 n_left_to_node; - u32 vectors = 0; - - ASSERT (fq); - ASSERT (vm == vlib_global_main.vlib_mains[thread_id]); - - if (PREDICT_FALSE (fqm->node_index == ~0)) - return 0; - /* - * Gather trace data for frame queues - */ - if (PREDICT_FALSE (fq->trace)) - { - frame_queue_trace_t *fqt; - frame_queue_nelt_counter_t *fqh; - u32 elix; - - fqt = &fqm->frame_queue_traces[thread_id]; - - fqt->nelts = fq->nelts; - fqt->head = fq->head; - fqt->head_hint = fq->head_hint; - fqt->tail = fq->tail; - fqt->threshold = fq->vector_threshold; - fqt->n_in_use = fqt->tail - fqt->head; - if (fqt->n_in_use >= fqt->nelts) - { - // if beyond max then use max - fqt->n_in_use = fqt->nelts - 1; - } - - /* Record the number of elements in use in the histogram */ - fqh = &fqm->frame_queue_histogram[thread_id]; - fqh->count[fqt->n_in_use]++; - - /* Record a snapshot of the elements in use */ - for (elix = 0; elix < fqt->nelts; elix++) - { - elt = fq->elts + ((fq->head + 1 + elix) & (fq->nelts - 1)); - if (1 || elt->valid) - { - fqt->n_vectors[elix] = elt->n_vectors; - } - } - fqt->written = 1; - } - - while (1) - { - vlib_buffer_t *b; - if (fq->head == fq->tail) - { - fq->head_hint = fq->head; - return processed; - } - - elt = fq->elts + ((fq->head + 1) & (fq->nelts - 1)); - - if (!elt->valid) - { - fq->head_hint = fq->head; - return processed; - } - - from = elt->buffer_index; - msg_type = elt->msg_type; - - ASSERT (msg_type == VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME); - ASSERT (elt->n_vectors <= VLIB_FRAME_SIZE); - - f = vlib_get_frame_to_node (vm, fqm->node_index); - - /* If the first vector is traced, set the frame trace flag */ - b = vlib_get_buffer (vm, from[0]); - if (b->flags & VLIB_BUFFER_IS_TRACED) - f->frame_flags |= VLIB_NODE_FLAG_TRACE; - - to = vlib_frame_vector_args (f); - - n_left_to_node = elt->n_vectors; - - while (n_left_to_node >= 4) - { - to[0] = from[0]; - to[1] = from[1]; - to[2] = from[2]; - to[3] = from[3]; - to += 4; - from += 4; - n_left_to_node -= 4; - } - - while (n_left_to_node > 0) - { - to[0] = from[0]; - to++; - from++; - n_left_to_node--; - } - - vectors += elt->n_vectors; - f->n_vectors = elt->n_vectors; - vlib_put_frame_to_node (vm, fqm->node_index, f); - - elt->valid = 0; - elt->n_vectors = 0; - elt->msg_type = 0xfefefefe; - CLIB_MEMORY_BARRIER (); - fq->head++; - processed++; - - /* - * Limit the number of packets pushed into the graph - */ - if (vectors >= fq->vector_threshold) - { - fq->head_hint = fq->head; - return processed; - } - } - ASSERT (0); - return processed; -} - void vlib_worker_thread_fn (void *arg) { diff --git a/src/vlib/threads.h b/src/vlib/threads.h index 1da9d22234e..9d9d387d155 100644 --- a/src/vlib/threads.h +++ b/src/vlib/threads.h @@ -179,9 +179,6 @@ int vlib_frame_queue_enqueue (vlib_main_t * vm, u32 node_runtime_index, u32 frame_queue_index, vlib_frame_t * frame, vlib_frame_queue_msg_type_t type); -int -vlib_frame_queue_dequeue (vlib_main_t * vm, vlib_frame_queue_main_t * fqm); - void vlib_worker_thread_node_runtime_update (void); void vlib_create_worker_threads (vlib_main_t * vm, int n, -- 2.16.6