1 /* SPDX-License-Identifier: Apache-2.0
2 * Copyright(c) 2021 Cisco Systems, Inc.
7 void __clib_section (".vlib_buffer_enqueue_to_next_fn")
8 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_next_fn)
9 (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 *nexts,
12 u32 *to_next, n_left_to_next, max;
15 next_index = nexts[0];
16 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
17 max = clib_min (n_left_to_next, count);
22 if ((nexts[0] != next_index) || n_left_to_next == 0)
24 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
25 next_index = nexts[0];
26 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
27 max = clib_min (n_left_to_next, count);
29 #if defined(CLIB_HAVE_VEC512)
30 u16x32 next32 = CLIB_MEM_OVERFLOW_LOAD (u16x32_load_unaligned, nexts);
31 next32 = (next32 == u16x32_splat (next32[0]));
32 u64 bitmap = u16x32_msb_mask (next32);
33 n_enqueued = count_trailing_zeros (~bitmap);
34 #elif defined(CLIB_HAVE_VEC256)
35 u16x16 next16 = CLIB_MEM_OVERFLOW_LOAD (u16x16_load_unaligned, nexts);
36 next16 = (next16 == u16x16_splat (next16[0]));
37 u64 bitmap = u8x32_msb_mask ((u8x32) next16);
38 n_enqueued = count_trailing_zeros (~bitmap) / 2;
39 #elif defined(CLIB_HAVE_VEC128) && defined(CLIB_HAVE_VEC128_MSB_MASK)
40 u16x8 next8 = CLIB_MEM_OVERFLOW_LOAD (u16x8_load_unaligned, nexts);
41 next8 = (next8 == u16x8_splat (next8[0]));
42 u64 bitmap = u8x16_msb_mask ((u8x16) next8);
43 n_enqueued = count_trailing_zeros (~bitmap) / 2;
48 x |= next_index ^ nexts[1];
49 x |= next_index ^ nexts[2];
50 x |= next_index ^ nexts[3];
51 n_enqueued = (x == 0) ? 4 : 1;
57 if (PREDICT_FALSE (n_enqueued > max))
60 #ifdef CLIB_HAVE_VEC512
63 vlib_buffer_copy_indices (to_next, buffers, 32);
74 #ifdef CLIB_HAVE_VEC256
77 vlib_buffer_copy_indices (to_next, buffers, 16);
88 #ifdef CLIB_HAVE_VEC128
91 vlib_buffer_copy_indices (to_next, buffers, 8);
104 vlib_buffer_copy_indices (to_next, buffers, 4);
115 to_next[0] = buffers[0];
125 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
128 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_next_fn);
130 void __clib_section (".vlib_buffer_enqueue_to_single_next_fn")
131 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_single_next_fn)
132 (vlib_main_t *vm, vlib_node_runtime_t *node, u32 *buffers, u16 next_index,
135 u32 *to_next, n_left_to_next, n_enq;
137 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
139 if (PREDICT_TRUE (n_left_to_next >= count))
141 vlib_buffer_copy_indices (to_next, buffers, count);
142 n_left_to_next -= count;
143 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
147 n_enq = n_left_to_next;
149 vlib_buffer_copy_indices (to_next, buffers, n_enq);
150 n_left_to_next -= n_enq;
152 if (PREDICT_FALSE (count > n_enq))
157 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
158 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
159 n_enq = clib_min (n_left_to_next, count);
162 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
165 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_single_next_fn);
167 u32 __clib_section (".vlib_buffer_enqueue_to_thread_fn")
168 CLIB_MULTIARCH_FN (vlib_buffer_enqueue_to_thread_fn)
169 (vlib_main_t *vm, u32 frame_queue_index, u32 *buffer_indices,
170 u16 *thread_indices, u32 n_packets, int drop_on_congestion)
172 vlib_thread_main_t *tm = vlib_get_thread_main ();
173 vlib_frame_queue_main_t *fqm;
174 vlib_frame_queue_per_thread_data_t *ptd;
175 u32 n_left = n_packets;
176 u32 drop_list[VLIB_FRAME_SIZE], *dbi = drop_list, n_drop = 0;
177 vlib_frame_queue_elt_t *hf = 0;
178 u32 n_left_to_next_thread = 0, *to_next_thread = 0;
179 u32 next_thread_index, current_thread_index = ~0;
182 fqm = vec_elt_at_index (tm->frame_queue_mains, frame_queue_index);
183 ptd = vec_elt_at_index (fqm->per_thread_data, vm->thread_index);
187 next_thread_index = thread_indices[0];
189 if (next_thread_index != current_thread_index)
191 if (drop_on_congestion &&
192 is_vlib_frame_queue_congested (
193 frame_queue_index, next_thread_index, fqm->queue_hi_thresh,
194 ptd->congested_handoff_queue_by_thread_index))
196 dbi[0] = buffer_indices[0];
203 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;
205 hf = vlib_get_worker_handoff_queue_elt (
206 frame_queue_index, next_thread_index,
207 ptd->handoff_queue_elt_by_thread_index);
209 n_left_to_next_thread = VLIB_FRAME_SIZE - hf->n_vectors;
210 to_next_thread = &hf->buffer_index[hf->n_vectors];
211 current_thread_index = next_thread_index;
214 to_next_thread[0] = buffer_indices[0];
216 n_left_to_next_thread--;
218 if (n_left_to_next_thread == 0)
220 hf->n_vectors = VLIB_FRAME_SIZE;
221 vlib_put_frame_queue_elt (hf);
222 vlib_get_main_by_index (current_thread_index)->check_frame_queues =
224 current_thread_index = ~0;
225 ptd->handoff_queue_elt_by_thread_index[next_thread_index] = 0;
237 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_thread;
239 /* Ship frames to the thread nodes */
240 for (i = 0; i < vec_len (ptd->handoff_queue_elt_by_thread_index); i++)
242 if (ptd->handoff_queue_elt_by_thread_index[i])
244 hf = ptd->handoff_queue_elt_by_thread_index[i];
246 * It works better to let the handoff node
247 * rate-adapt, always ship the handoff queue element.
249 if (1 || hf->n_vectors == hf->last_n_vectors)
251 vlib_put_frame_queue_elt (hf);
252 vlib_get_main_by_index (i)->check_frame_queues = 1;
253 ptd->handoff_queue_elt_by_thread_index[i] = 0;
256 hf->last_n_vectors = hf->n_vectors;
258 ptd->congested_handoff_queue_by_thread_index[i] =
259 (vlib_frame_queue_t *) (~0);
262 if (drop_on_congestion && n_drop)
263 vlib_buffer_free (vm, drop_list, n_drop);
265 return n_packets - n_drop;
268 CLIB_MARCH_FN_REGISTRATION (vlib_buffer_enqueue_to_thread_fn);
270 #ifndef CLIB_MARCH_VARIANT
271 vlib_buffer_func_main_t vlib_buffer_func_main;
273 static clib_error_t *
274 vlib_buffer_funcs_init (vlib_main_t *vm)
276 vlib_buffer_func_main_t *bfm = &vlib_buffer_func_main;
277 bfm->buffer_enqueue_to_next_fn =
278 CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_next_fn);
279 bfm->buffer_enqueue_to_single_next_fn =
280 CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_single_next_fn);
281 bfm->buffer_enqueue_to_thread_fn =
282 CLIB_MARCH_FN_POINTER (vlib_buffer_enqueue_to_thread_fn);
286 VLIB_INIT_FUNCTION (vlib_buffer_funcs_init);