X-Git-Url: https://gerrit.fd.io/r/gitweb?a=blobdiff_plain;f=src%2Fvlib%2Fnode_funcs.h;h=df8ec7c40e6ce7654a1597f4549a0f640061b2cc;hb=1c8c6d302af494c963842de06910d3c1c8187e42;hp=c9ff93d6124444365b3906342cd64822629d8a5f;hpb=11965c7b47670da7d02eb2c543be52008c66848e;p=vpp.git diff --git a/src/vlib/node_funcs.h b/src/vlib/node_funcs.h index c9ff93d6124..df8ec7c40e6 100644 --- a/src/vlib/node_funcs.h +++ b/src/vlib/node_funcs.h @@ -47,6 +47,33 @@ #include #include +#include + +#ifdef CLIB_SANITIZE_ADDR +#include +#endif + +static_always_inline void +vlib_process_start_switch_stack (vlib_main_t * vm, vlib_process_t * p) +{ +#ifdef CLIB_SANITIZE_ADDR + void *stack = p ? (void *) p->stack : vlib_thread_stacks[vm->thread_index]; + u32 stack_bytes = p ? p->log2_n_stack_bytes : VLIB_THREAD_STACK_SIZE; + __sanitizer_start_switch_fiber (&vm->asan_stack_save, stack, stack_bytes); +#endif +} + +static_always_inline void +vlib_process_finish_switch_stack (vlib_main_t * vm) +{ +#ifdef CLIB_SANITIZE_ADDR + const void *bottom_old; + size_t size_old; + + __sanitizer_finish_switch_fiber (&vm->asan_stack_save, &bottom_old, + &size_old); +#endif +} /** \brief Get vlib node by index. @warning This function will ASSERT if @c i is out of range. @@ -175,6 +202,10 @@ vlib_node_set_state (vlib_main_t * vm, u32 node_index, nm->input_node_counts_by_state[new_state] += 1; } + if (PREDICT_FALSE (r->state == VLIB_NODE_STATE_DISABLED)) + vlib_node_runtime_perf_counter (vm, r, 0, 0, 0, + VLIB_NODE_RUNTIME_PERF_RESET); + n->state = new_state; r->state = new_state; } @@ -194,14 +225,40 @@ vlib_node_get_state (vlib_main_t * vm, u32 node_index) } always_inline void -vlib_node_set_interrupt_pending (vlib_main_t * vm, u32 node_index) +vlib_node_set_flag (vlib_main_t *vm, u32 node_index, u16 flag, u8 enable) +{ + vlib_node_runtime_t *r; + vlib_node_t *n; + + n = vlib_get_node (vm, node_index); + r = vlib_node_get_runtime (vm, node_index); + + if (enable) + { + n->flags |= flag; + r->flags |= flag; + } + else + { + n->flags &= ~flag; + r->flags &= ~flag; + } +} + +always_inline void +vlib_node_set_interrupt_pending (vlib_main_t *vm, u32 node_index) { vlib_node_main_t *nm = &vm->node_main; vlib_node_t *n = vec_elt (nm->nodes, node_index); + ASSERT (n->type == VLIB_NODE_TYPE_INPUT); - clib_spinlock_lock_if_init (&nm->pending_interrupt_lock); - vec_add1 (nm->pending_interrupt_node_runtime_indices, n->runtime_index); - clib_spinlock_unlock_if_init (&nm->pending_interrupt_lock); + + if (vm != vlib_get_main ()) + clib_interrupt_set_atomic (nm->interrupts, n->runtime_index); + else + clib_interrupt_set (nm->interrupts, n->runtime_index); + + __atomic_store_n (nm->pending_interrupts, 1, __ATOMIC_RELEASE); } always_inline vlib_process_t * @@ -212,32 +269,10 @@ vlib_get_process_from_node (vlib_main_t * vm, vlib_node_t * node) return vec_elt (nm->processes, node->runtime_index); } -/* Fetches frame with given handle. */ -always_inline vlib_frame_t * -vlib_get_frame_no_check (vlib_main_t * vm, uword frame_index) -{ - vlib_frame_t *f; - f = vm->heap_aligned_base + (frame_index * VLIB_FRAME_ALIGN); - return f; -} - -always_inline u32 -vlib_frame_index_no_check (vlib_main_t * vm, vlib_frame_t * f) -{ - uword i; - - ASSERT (((uword) f & (VLIB_FRAME_ALIGN - 1)) == 0); - - i = ((u8 *) f - (u8 *) vm->heap_aligned_base); - ASSERT ((i / VLIB_FRAME_ALIGN) <= 0xFFFFFFFFULL); - - return i / VLIB_FRAME_ALIGN; -} - always_inline vlib_frame_t * -vlib_get_frame (vlib_main_t * vm, uword frame_index) +vlib_get_frame (vlib_main_t * vm, vlib_frame_t * f) { - vlib_frame_t *f = vlib_get_frame_no_check (vm, frame_index); + ASSERT (f != NULL); ASSERT (f->frame_flags & VLIB_FRAME_IS_ALLOCATED); return f; } @@ -248,24 +283,6 @@ vlib_frame_no_append (vlib_frame_t * f) f->frame_flags |= VLIB_FRAME_NO_APPEND; } -always_inline u32 -vlib_frame_index (vlib_main_t * vm, vlib_frame_t * f) -{ - uword i = vlib_frame_index_no_check (vm, f); - ASSERT (vlib_get_frame (vm, i) == f); - return i; -} - -/* Byte alignment for vector arguments. */ -#define VLIB_FRAME_VECTOR_ALIGN (1 << 4) - -always_inline u32 -vlib_frame_vector_byte_offset (u32 scalar_size) -{ - return round_pow2 (sizeof (vlib_frame_t) + scalar_size, - VLIB_FRAME_VECTOR_ALIGN); -} - /** \brief Get pointer to frame vector data. @param f vlib_frame_t pointer @return pointer to first vector element in frame @@ -273,7 +290,19 @@ vlib_frame_vector_byte_offset (u32 scalar_size) always_inline void * vlib_frame_vector_args (vlib_frame_t * f) { - return (void *) f + vlib_frame_vector_byte_offset (f->scalar_size); + ASSERT (f->vector_offset); + return (void *) f + f->vector_offset; +} + +/** \brief Get pointer to frame vector aux data. + @param f vlib_frame_t pointer + @return pointer to first vector aux data element in frame +*/ +always_inline void * +vlib_frame_aux_args (vlib_frame_t *f) +{ + ASSERT (f->aux_offset); + return (void *) f + f->aux_offset; } /** \brief Get pointer to frame scalar data. @@ -287,7 +316,8 @@ vlib_frame_vector_args (vlib_frame_t * f) always_inline void * vlib_frame_scalar_args (vlib_frame_t * f) { - return vlib_frame_vector_args (f) - f->scalar_size; + ASSERT (f->scalar_offset); + return (void *) f + f->scalar_offset; } always_inline vlib_next_frame_t * @@ -432,6 +462,13 @@ vlib_current_process (vlib_main_t * vm) return vlib_get_current_process (vm)->node_runtime.node_index; } +always_inline u32 +vlib_get_current_process_node_index (vlib_main_t * vm) +{ + vlib_process_t *process = vlib_get_current_process (vm); + return process->node_runtime.node_index; +} + /** Returns TRUE if a process suspend time is less than 10us @param dt - remaining poll time in seconds @returns 1 if dt < 10e-6, 0 otherwise @@ -464,8 +501,11 @@ vlib_process_suspend (vlib_main_t * vm, f64 dt) { /* expiration time in 10us ticks */ p->resume_clock_interval = dt * 1e5; + vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); } + else + vlib_process_finish_switch_stack (vm); return r; } @@ -564,7 +604,7 @@ vlib_process_get_events (vlib_main_t * vm, uword ** data_vector) l = _vec_len (p->pending_event_data_by_type_index[t]); if (data_vector) vec_add (*data_vector, p->pending_event_data_by_type_index[t], l); - _vec_len (p->pending_event_data_by_type_index[t]) = 0; + vec_set_len (p->pending_event_data_by_type_index[t], 0); et = pool_elt_at_index (p->event_type_pool, t); @@ -588,7 +628,7 @@ vlib_process_get_events_helper (vlib_process_t * p, uword t, l = _vec_len (p->pending_event_data_by_type_index[t]); if (data_vector) vec_add (*data_vector, p->pending_event_data_by_type_index[t], l); - _vec_len (p->pending_event_data_by_type_index[t]) = 0; + vec_set_len (p->pending_event_data_by_type_index[t], 0); vlib_process_maybe_free_event_type (p, t); @@ -633,8 +673,13 @@ vlib_process_wait_for_event (vlib_main_t * vm) r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) - clib_longjmp (&p->return_longjmp, - VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + { + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, + VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); } return p->non_empty_event_type_bitmap; @@ -657,8 +702,13 @@ vlib_process_wait_for_one_time_event (vlib_main_t * vm, r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) - clib_longjmp (&p->return_longjmp, - VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + { + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, + VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); } return vlib_process_get_events_helper (p, with_type_index, data_vector); @@ -681,8 +731,13 @@ vlib_process_wait_for_event_with_type (vlib_main_t * vm, r = clib_setjmp (&p->resume_longjmp, VLIB_PROCESS_RESUME_LONGJMP_SUSPEND); if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) - clib_longjmp (&p->return_longjmp, - VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + { + vlib_process_start_switch_stack (vm, 0); + clib_longjmp (&p->return_longjmp, + VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); + } + else + vlib_process_finish_switch_stack (vm); /* See if unknown event type has been signaled now. */ if (!h) @@ -723,8 +778,11 @@ vlib_process_wait_for_event_or_clock (vlib_main_t * vm, f64 dt) if (r == VLIB_PROCESS_RESUME_LONGJMP_SUSPEND) { p->resume_clock_interval = dt * 1e5; + vlib_process_start_switch_stack (vm, 0); clib_longjmp (&p->return_longjmp, VLIB_PROCESS_RETURN_LONGJMP_SUSPEND); } + else + vlib_process_finish_switch_stack (vm); /* Return amount of time still left to sleep. If <= 0 then we've been waken up by the clock (and not an event). */ @@ -777,7 +835,7 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, uword n_data_elts, uword n_data_elt_bytes) { uword p_flags, add_to_pending, delete_from_wheel; - void *data_to_be_written_by_caller; + u8 *data_to_be_written_by_caller; ASSERT (n->type == VLIB_NODE_TYPE_PROCESS); @@ -787,22 +845,20 @@ vlib_process_signal_event_helper (vlib_node_main_t * nm, /* Resize data vector and return caller's data to be written. */ { - void *data_vec = p->pending_event_data_by_type_index[t]; + u8 *data_vec = p->pending_event_data_by_type_index[t]; uword l; if (!data_vec && vec_len (nm->recycled_event_data_vectors)) { data_vec = vec_pop (nm->recycled_event_data_vectors); - _vec_len (data_vec) = 0; + vec_reset_length (data_vec); } l = vec_len (data_vec); - data_vec = _vec_resize (data_vec, - /* length_increment */ n_data_elts, - /* total size after increment */ - (l + n_data_elts) * n_data_elt_bytes, - /* header_bytes */ 0, /* data_align */ 0); + data_vec = + _vec_realloc (data_vec, l + n_data_elts, n_data_elt_bytes, + /* header_bytes */ 0, /* data_align */ 0, /* heap */ 0); p->pending_event_data_by_type_index[t] = data_vec; data_to_be_written_by_caller = data_vec + l * n_data_elt_bytes; @@ -1153,7 +1209,11 @@ void vlib_node_rename (vlib_main_t * vm, u32 node_index, char *fmt, ...); /* Register new packet processing node. Nodes can be registered dynamically via this call or statically via the VLIB_REGISTER_NODE macro. */ -u32 vlib_register_node (vlib_main_t * vm, vlib_node_registration_t * r); +u32 vlib_register_node (vlib_main_t *vm, vlib_node_registration_t *r, + char *fmt, ...); + +/* Register all node function variants */ +void vlib_register_all_node_march_variants (vlib_main_t *vm); /* Register all static nodes registered via VLIB_REGISTER_NODE. */ void vlib_register_all_static_nodes (vlib_main_t * vm); @@ -1163,6 +1223,12 @@ void vlib_start_process (vlib_main_t * vm, uword process_index); /* Sync up runtime and main node stats. */ void vlib_node_sync_stats (vlib_main_t * vm, vlib_node_t * n); +void vlib_node_runtime_sync_stats (vlib_main_t *vm, vlib_node_runtime_t *r, + uword n_calls, uword n_vectors, + uword n_clocks); +void vlib_node_runtime_sync_stats_node (vlib_node_t *n, vlib_node_runtime_t *r, + uword n_calls, uword n_vectors, + uword n_clocks); /* Node graph initialization function. */ clib_error_t *vlib_node_main_init (vlib_main_t * vm); @@ -1196,6 +1262,115 @@ vlib_node_increment_counter (vlib_main_t * vm, u32 node_index, u32 vlib_process_create (vlib_main_t * vm, char *name, vlib_node_function_t * f, u32 log2_n_stack_bytes); +always_inline int +vlib_node_set_dispatch_wrapper (vlib_main_t *vm, vlib_node_function_t *fn) +{ + if (fn && vm->dispatch_wrapper_fn) + return 1; + vm->dispatch_wrapper_fn = fn; + return 0; +} + +int vlib_node_set_march_variant (vlib_main_t *vm, u32 node_index, + clib_march_variant_type_t march_variant); + +vlib_node_function_t * +vlib_node_get_preferred_node_fn_variant (vlib_main_t *vm, + vlib_node_fn_registration_t *regs); + +/* + * vlib_frame_bitmap functions + */ + +#define VLIB_FRAME_BITMAP_N_UWORDS \ + (((VLIB_FRAME_SIZE + uword_bits - 1) & ~(uword_bits - 1)) / uword_bits) + +typedef uword vlib_frame_bitmap_t[VLIB_FRAME_BITMAP_N_UWORDS]; + +static_always_inline void +vlib_frame_bitmap_init (uword *bmp, u32 n_first_bits_set) +{ + u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS; + while (n_first_bits_set >= (sizeof (uword) * 8) && n_left) + { + bmp++[0] = ~0; + n_first_bits_set -= sizeof (uword) * 8; + n_left--; + } + + if (n_first_bits_set && n_left) + { + bmp++[0] = pow2_mask (n_first_bits_set); + n_left--; + } + + while (n_left--) + bmp++[0] = 0; +} + +static_always_inline void +vlib_frame_bitmap_clear (uword *bmp) +{ + u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS; + while (n_left--) + bmp++[0] = 0; +} + +static_always_inline void +vlib_frame_bitmap_xor (uword *bmp, uword *bmp2) +{ + u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS; + while (n_left--) + bmp++[0] ^= bmp2++[0]; +} + +static_always_inline void +vlib_frame_bitmap_or (uword *bmp, uword *bmp2) +{ + u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS; + while (n_left--) + bmp++[0] |= bmp2++[0]; +} + +static_always_inline void +vlib_frame_bitmap_and (uword *bmp, uword *bmp2) +{ + u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS; + while (n_left--) + bmp++[0] &= bmp2++[0]; +} + +static_always_inline u32 +vlib_frame_bitmap_count_set_bits (uword *bmp) +{ + u32 n_left = VLIB_FRAME_BITMAP_N_UWORDS; + u32 count = 0; + while (n_left--) + count += count_set_bits (bmp++[0]); + return count; +} + +static_always_inline int +vlib_frame_bitmap_find_first_set (uword *bmp) +{ + uword *b = bmp; + while (b[0] == 0) + { + ASSERT (b - bmp < VLIB_FRAME_BITMAP_N_UWORDS); + b++; + } + + return (b - bmp) * uword_bits + get_lowest_set_bit_index (b[0]); +} + +#define foreach_vlib_frame_bitmap_set_bit_index(i, v) \ + for (uword _off = 0; _off < ARRAY_LEN (v); _off++) \ + for (uword _tmp = \ + (v[_off]) + 0 * (uword) (i = _off * uword_bits + \ + get_lowest_set_bit_index (v[_off])); \ + _tmp; i = _off * uword_bits + get_lowest_set_bit_index ( \ + _tmp = clear_lowest_set_bit (_tmp))) + #endif /* included_vlib_node_funcs_h */ /*