X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fnode_funcs.h;h=b33f4960a90d5d45b29cb73cebca493dbcf08b30;hb=8b60fb0fe;hp=8ccfc43802053287360b7aed365d94f946733537;hpb=80f54e20270ed0628ee725e3e3c515731a0188f2;p=vpp.git diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index 8ccfc438020..b33f4960a90 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -46,6 +46,33 @@ #define included_vlib_node_funcs_h #include +#include + +#ifdef CLIB_SANITIZE_ADDR +#include +#endif + +static_always_inline void +vlib_process_start_switch_stack (vlib_main_t * vm, vlib_process_t * p) +{ +#ifdef CLIB_SANITIZE_ADDR + void *stack = p ? (void *) p->stack : vlib_thread_stacks[vm->thread_index]; + u32 stack_bytes = p ? p->log2_n_stack_bytes : VLIB_THREAD_STACK_SIZE; + __sanitizer_start_switch_fiber (&vm->asan_stack_save, stack, stack_bytes); +#endif +} + +static_always_inline void +vlib_process_finish_switch_stack (vlib_main_t * vm) +{ +#ifdef CLIB_SANITIZE_ADDR + const void *bottom_old; + size_t size_old; + + __sanitizer_finish_switch_fiber (&vm->asan_stack_save, &bottom_old, + &size_old); +#endif +} /** \brief Get vlib node by index. @warning This function will ASSERT if @c i is out of range. @@ -134,7 +161,8 @@ vlib_node_set_runtime_data (vlib_main_t * vm, u32 node_index, STRUCT_OFFSET_OF (vlib_node_runtime_t, runtime_data)); if (vec_len (n->runtime_data) > 0) - clib_memcpy (r->runtime_data, n->runtime_data, vec_len (n->runtime_data)); + clib_memcpy_fast (r->runtime_data, n->runtime_data, + vec_len (n->runtime_data)); } /** \brief Set node dispatch state. @@ -173,17 +201,60 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index, nm->input_node_counts_by_state[new_state] += 1; } + if (PREDICT_FALSE (r->state == VLIB_NODE_STATE_DISABLED)) + vlib_node_runtime_perf_counter (vm, r, 0, 0, 0, + VLIB_NODE_RUNTIME_PERF_RESET); + n->state = new_state; r->state = new_state; } +/** \brief Get node dispatch state. + @param vm vlib_main_t pointer, varies by thread + @param node_index index of the node + @return state for node, see vlib_node_state_t +*/ +always_inline vlib_node_state_t +vlib_node_get_state (vlib_main_t * vm, u32 node_index) +{ + vlib_node_main_t *nm = &vm->node_main; + vlib_node_t *n; + n = vec_elt (nm->nodes, node_index); + return n->state; +} + always_inline void -vlib_node_set_interrupt_pending (vlib_main_t * vm, u32 node_index) +vlib_node_set_interrupt_pending_with_data (vlib_main_t * vm, u32 node_index, + u32 data) { vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vec_elt (nm->nodes, node_index); + vlib_node_interrupt_t *i; ASSERT (n->type == VLIB_NODE_TYPE_INPUT); - vec_add1 (nm->pending_interrupt_node_runtime_indices, n->runtime_index); + + if (vm == vlib_get_main ()) + { + /* local thread */ + vec_add2 (nm->pending_local_interrupts, i, 1); + i->node_runtime_index = n->runtime_index; + i->data = data; + } + else + { + /* remote thread */ + clib_spinlock_lock (&nm->pending_interrupt_lock); + vec_add2 (nm->pending_remote_interrupts, i, 1); + i->node_runtime_index = n->runtime_index; + i->data = data; + *nm->pending_remote_interrupts_notify = 1; + clib_spinlock_unlock (&nm->pending_interrupt_lock); + } +} + +always_inline void +vlib_node_set_interrupt_pending (vlib_main_t * vm, u32 node_index) +{ + vlib_node_set_interrupt_pending_with_data (vm, node_index, 0); } always_inline vlib_process_t * @@ -194,45 +265,18 @@ vlib_get_process_from_node (vlib_main_t * vm, vlib_node_t * node) return vec_elt (nm->processes, node->runtime_index); } -/* Fetches frame with given handle. */ always_inline vlib_frame_t * -vlib_get_frame_no_check (vlib_main_t * vm, uword frame_index) +vlib_get_frame (vlib_main_t * vm, vlib_frame_t * f) { - vlib_frame_t *f; - u32 cpu_index = frame_index & VLIB_CPU_MASK; - u32 offset = frame_index & VLIB_OFFSET_MASK; - vm = vlib_mains[cpu_index]; - f = vm->heap_base + offset; + ASSERT (f != NULL); + ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); return f; } -always_inline u32 -vlib_frame_index_no_check (vlib_main_t * vm, vlib_frame_t * f) -{ - u32 i; - - ASSERT (((uword) f & VLIB_CPU_MASK) == 0); - - vm = vlib_mains[f->cpu_index]; - - i = ((u8 *) f - (u8 *) vm->heap_base); - return i | f->cpu_index; -} - -always_inline vlib_frame_t * -vlib_get_frame (vlib_main_t * vm, uword frame_index) -{ - vlib_frame_t *f = vlib_get_frame_no_check (vm, frame_index); - ASSERT (f->flags & VLIB_FRAME_IS_ALLOCATED); - return f; -} - -always_inline u32 -vlib_frame_index (vlib_main_t * vm, vlib_frame_t * f) +always_inline void +vlib_frame_no_append (vlib_frame_t * f) { - uword i = vlib_frame_index_no_check (vm, f); - ASSERT (vlib_get_frame (vm, i) == f); - return i; + f->frame_flags |= VLIB_FRAME_NO_APPEND; } /* Byte alignment for vector arguments. */ @@ -257,9 +301,6 @@ vlib_frame_vector_args (vlib_frame_t * f) /** \brief Get pointer to frame scalar data. - @warning This is almost certainly not the function you wish to call. - See @ref vlib_frame_vector_args instead. - @param f vlib_frame_t pointer @return arbitrary node scalar data @@ -267,7 +308,7 @@ vlib_frame_vector_args (vlib_frame_t * f) @sa vlib_frame_vector_args */ always_inline void * -vlib_frame_args (vlib_frame_t * f) +vlib_frame_scalar_args (vlib_frame_t * f) { return vlib_frame_vector_args (f) - f->scalar_size; } @@ -393,33 +434,42 @@ vlib_frame_t *vlib_get_frame_to_node (vlib_main_t * vm, u32 to_node_index); void vlib_put_frame_to_node (vlib_main_t * vm, u32 to_node_index, vlib_frame_t * f); +always_inline uword +vlib_in_process_context (vlib_main_t * vm) +{ + return vm->node_main.current_process_index != ~0; +} + always_inline vlib_process_t * vlib_get_current_process (vlib_main_t * vm) { vlib_node_main_t *nm = &vm->node_main; - return vec_elt (nm->processes, nm->current_process_index); + if (vlib_in_process_context (vm)) + return vec_elt (nm->processes, nm->current_process_index); + return 0; } always_inline uword -vlib_in_process_context (vlib_main_t * vm) +vlib_current_process (vlib_main_t * vm) { - return vm->node_main.current_process_index != ~0; + return vlib_get_current_process (vm)->node_runtime.node_index; } -always_inline uword -vlib_current_process (vlib_main_t * vm) +always_inline u32 +vlib_get_current_process_node_index (vlib_main_t * vm) { - return vlib_get_current_process (vm)->node_runtime.node_index; + vlib_process_t *process = vlib_get_current_process (vm); + return process->node_runtime.node_index; } -/** Returns TRUE if a process suspend time is less than 1us +/** Returns TRUE if a process suspend time is less than 10us @param dt - remaining poll time in seconds - @returns 1 if dt < 1e-6, 0 otherwise + @returns 1 if dt < 10e-6, 0 otherwise */ always_inline uword vlib_process_suspend_time_is_zero (f64 dt) { - return dt < 1e-6; + return dt < 10e-6; } /** Suspend a vlib cooperative multi-tasking thread for a period of time @@ -434,7 +484,6 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt) uword r; vlib_node_main_t *nm = &vm->node_main; vlib_process_t *p = vec_elt (nm->processes, nm->current_process_index); - u64 dt_cpu = dt * vm->clib_time.clocks_per_second; if (vlib_process_suspend_time_is_zero (dt)) return VLIB_PROCESS_RESUME_LONGJMP_RESUME; @@ -443,9 +492,13 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt) r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { - p->resume_cpu_time = clib_cpu_time_now () + dt_cpu; + /* expiration time in 10us ticks */ + p->resume_clock_interval = dt * 1e5; + vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); } + else + vlib_process_finish_switch_stack (vm); return r; } @@ -613,8 +666,13 @@ vlib_process_wait_for_event (vlib_main_t * vm) r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) - clib_longjmp (&p->return_longjmp, - VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + { + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, + VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); } return p->non_empty_event_type_bitmap; @@ -637,8 +695,13 @@ vlib_process_wait_for_one_time_event (vlib_main_t * vm, r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) - clib_longjmp (&p->return_longjmp, - VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + { + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, + VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); } return vlib_process_get_events_helper (p, with_type_index, data_vector); @@ -661,8 +724,13 @@ vlib_process_wait_for_event_with_type (vlib_main_t * vm, r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) - clib_longjmp (&p->return_longjmp, - VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + { + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, + VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); /* See if unknown event type has been signaled now. */ if (!h) @@ -702,10 +770,12 @@ vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt) r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { - p->resume_cpu_time = (clib_cpu_time_now () - + (dt * vm->clib_time.clocks_per_second)); + p->resume_clock_interval = dt * 1e5; + vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); } + else + vlib_process_finish_switch_stack (vm); /* Return amount of time still left to sleep. If <= 0 then we've been waken up by the clock (and not an event). */ @@ -760,6 +830,8 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, uword p_flags, add_to_pending, delete_from_wheel; void *data_to_be_written_by_caller; + ASSERT (n->type == VLIB_NODE_TYPE_PROCESS); + ASSERT (!pool_is_free_index (p->event_type_pool, t)); vec_validate (p->pending_event_data_by_type_index, t); @@ -801,7 +873,15 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, { /* Waiting for both event and clock? */ if (p_flags & VLIB_PROCESS_IS_SUSPENDED_WAITING_FOR_EVENT) - delete_from_wheel = 1; + { + if (!TW (tw_timer_handle_is_free) + ((TWT (tw_timer_wheel) *) nm->timing_wheel, + p->stop_timer_handle)) + delete_from_wheel = 1; + else + /* timer just popped so process should already be on the list */ + add_to_pending = 0; + } else /* Waiting only for clock. Event will be queue and may be handled when timer expires. */ @@ -818,7 +898,8 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, p->flags = p_flags | VLIB_PROCESS_RESUME_PENDING; vec_add1 (nm->data_from_advancing_timing_wheel, x); if (delete_from_wheel) - timing_wheel_delete (&nm->timing_wheel, x); + TW (tw_timer_stop) ((TWT (tw_timer_wheel) *) nm->timing_wheel, + p->stop_timer_handle); } return data_to_be_written_by_caller; @@ -835,6 +916,9 @@ vlib_process_signal_event_data (vlib_main_t * vm, vlib_process_t *p = vec_elt (nm->processes, n->runtime_index); uword *h, t; + /* Must be in main thread */ + ASSERT (vlib_get_thread_index () == 0); + h = hash_get (p->event_type_index_by_type_opaque, type_opaque); if (!h) { @@ -879,7 +963,6 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, else { vlib_signal_timed_event_data_t *te; - u64 dt_cpu = dt * vm->clib_time.clocks_per_second; pool_get_aligned (nm->signal_timed_event_data_pool, te, sizeof (te[0])); @@ -895,10 +978,12 @@ vlib_process_signal_event_at_time (vlib_main_t * vm, te->process_node_index = n->runtime_index; te->event_type_index = t; - timing_wheel_insert (&nm->timing_wheel, clib_cpu_time_now () + dt_cpu, - vlib_timing_wheel_data_set_timed_event (te - - nm-> - signal_timed_event_data_pool)); + p->stop_timer_handle = + TW (tw_timer_start) ((TWT (tw_timer_wheel) *) nm->timing_wheel, + vlib_timing_wheel_data_set_timed_event + (te - nm->signal_timed_event_data_pool), + 0 /* timer_id */ , + (vlib_time_now (vm) + dt) * 1e5); /* Inline data big enough to hold event? */ if (te->n_data_bytes < sizeof (te->inline_event_data)) @@ -945,6 +1030,29 @@ vlib_process_signal_event_pointer (vlib_main_t * vm, d[0] = data; } +/** + * Signal event to process from any thread. + * + * When in doubt, use this. + */ +always_inline void +vlib_process_signal_event_mt (vlib_main_t * vm, + uword node_index, uword type_opaque, uword data) +{ + if (vlib_get_thread_index () != 0) + { + vlib_process_signal_event_mt_args_t args = { + .node_index = node_index, + .type_opaque = type_opaque, + .data = data, + }; + vlib_rpc_call_main_thread (vlib_process_signal_event_mt_helper, + (u8 *) & args, sizeof (args)); + } + else + vlib_process_signal_event (vm, node_index, type_opaque, data); +} + always_inline void vlib_process_signal_one_time_event (vlib_main_t * vm, uword node_index, @@ -962,7 +1070,7 @@ vlib_signal_one_time_waiting_process (vlib_main_t * vm, { vlib_process_signal_one_time_event (vm, p->node_index, p->one_time_event, /* data */ ~0); - memset (p, ~0, sizeof (p[0])); + clib_memset (p, ~0, sizeof (p[0])); } always_inline void @@ -1052,6 +1160,9 @@ vlib_node_vectors_per_main_loop_as_integer (vlib_main_t * vm, u32 node_index) void vlib_frame_free (vlib_main_t * vm, vlib_node_runtime_t * r, vlib_frame_t * f); +/* Return the edge index if present, ~0 otherwise */ +uword vlib_node_get_next (vlib_main_t * vm, uword node, uword next_node); + /* Add next node to given node in given slot. */ uword vlib_node_add_next_with_slot (vlib_main_t * vm, @@ -1076,6 +1187,14 @@ vlib_node_add_named_next (vlib_main_t * vm, uword node, char *name) return vlib_node_add_named_next_with_slot (vm, node, name, ~0); } +/** + * Get list of nodes + */ +void +vlib_node_get_nodes (vlib_main_t * vm, u32 max_threads, int include_stats, + int barrier_sync, vlib_node_t **** node_dupsp, + vlib_main_t *** stat_vmsp); + /* Query node given name. */ vlib_node_t *vlib_get_node_by_name (vlib_main_t * vm, u8 * name); @@ -1118,6 +1237,25 @@ vlib_node_increment_counter (vlib_main_t * vm, u32 node_index, em->counters[node_counter_base_index + counter_index] += increment; } +/** @brief Create a vlib process + * @param vm &vlib_global_main + * @param f the process node function + * @param log2_n_stack_bytes size of the process stack, defaults to 16K + * @return newly-create node index + * @warning call only on the main thread. Barrier sync required + */ +u32 vlib_process_create (vlib_main_t * vm, char *name, + vlib_node_function_t * f, u32 log2_n_stack_bytes); + +always_inline int +vlib_node_set_dispatch_wrapper (vlib_main_t *vm, vlib_node_function_t *fn) +{ + if (fn && vm->dispatch_wrapper_fn) + return 1; + vm->dispatch_wrapper_fn = fn; + return 0; +} + #endif /* included_vlib_node_funcs_h */ /*