2 * Copyright (c) 2015 Cisco and/or its affiliates.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at:
7 * http://www.apache.org/licenses/LICENSE-2.0
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
16 * buffer_node.h: VLIB buffer handling node helper macros/inlines
18 * Copyright (c) 2008 Eliot Dresselhaus
20 * Permission is hereby granted, free of charge, to any person obtaining
21 * a copy of this software and associated documentation files (the
22 * "Software"), to deal in the Software without restriction, including
23 * without limitation the rights to use, copy, modify, merge, publish,
24 * distribute, sublicense, and/or sell copies of the Software, and to
25 * permit persons to whom the Software is furnished to do so, subject to
26 * the following conditions:
28 * The above copyright notice and this permission notice shall be
29 * included in all copies or substantial portions of the Software.
31 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
32 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
33 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
34 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
35 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
36 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
37 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
40 #ifndef included_vlib_buffer_node_h
41 #define included_vlib_buffer_node_h
44 vlib buffer/node functions
47 /** \brief Finish enqueueing two buffers forward in the graph.
48 Standard dual loop boilerplate element. This is a MACRO,
49 with MULTIPLE SIDE EFFECTS. In the ideal case,
50 <code>next_index == next0 == next1</code>,
51 which means that the speculative enqueue at the top of the dual loop
52 has correctly dealt with both packets. In that case, the macro does
55 @param vm vlib_main_t pointer, varies by thread
56 @param node current node vlib_node_runtime_t pointer
57 @param next_index speculated next index used for both packets
58 @param to_next speculated vector pointer used for both packets
59 @param n_left_to_next number of slots left in speculated vector
60 @param bi0 first buffer index
61 @param bi1 second buffer index
62 @param next0 actual next index to be used for the first packet
63 @param next1 actual next index to be used for the second packet
65 @return @c next_index -- speculative next index to be used for future packets
66 @return @c to_next -- speculative frame to be used for future packets
67 @return @c n_left_to_next -- number of slots left in speculative frame
70 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
74 int enqueue_code = (next0 != next_index) + 2*(next1 != next_index); \
76 if (PREDICT_FALSE (enqueue_code != 0)) \
78 switch (enqueue_code) \
84 n_left_to_next += 1; \
85 vlib_set_next_frame_buffer (vm, node, next0, bi0); \
91 n_left_to_next += 1; \
92 vlib_set_next_frame_buffer (vm, node, next1, bi1); \
96 /* A B B or A B C */ \
98 n_left_to_next += 2; \
99 vlib_set_next_frame_buffer (vm, node, next0, bi0); \
100 vlib_set_next_frame_buffer (vm, node, next1, bi1); \
101 if (next0 == next1) \
103 vlib_put_next_frame (vm, node, next_index, \
105 next_index = next1; \
106 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
113 /** \brief Finish enqueueing four buffers forward in the graph.
114 Standard quad loop boilerplate element. This is a MACRO,
115 with MULTIPLE SIDE EFFECTS. In the ideal case,
116 <code>next_index == next0 == next1 == next2 == next3</code>,
117 which means that the speculative enqueue at the top of the quad loop
118 has correctly dealt with all four packets. In that case, the macro does
121 @param vm vlib_main_t pointer, varies by thread
122 @param node current node vlib_node_runtime_t pointer
123 @param next_index speculated next index used for both packets
124 @param to_next speculated vector pointer used for both packets
125 @param n_left_to_next number of slots left in speculated vector
126 @param bi0 first buffer index
127 @param bi1 second buffer index
128 @param bi2 third buffer index
129 @param bi3 fourth buffer index
130 @param next0 actual next index to be used for the first packet
131 @param next1 actual next index to be used for the second packet
132 @param next2 actual next index to be used for the third packet
133 @param next3 actual next index to be used for the fourth packet
135 @return @c next_index -- speculative next index to be used for future packets
136 @return @c to_next -- speculative frame to be used for future packets
137 @return @c n_left_to_next -- number of slots left in speculative frame
140 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
146 /* After the fact: check the [speculative] enqueue to "next" */ \
147 u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1) \
148 | (next_index ^ next2) | (next_index ^ next3); \
149 if (PREDICT_FALSE(fix_speculation)) \
153 n_left_to_next += 4; \
155 /* If bi0 belongs to "next", send it there */ \
156 if (next_index == next0) \
162 else /* send it where it needs to go */ \
163 vlib_set_next_frame_buffer (vm, node, next0, bi0); \
165 if (next_index == next1) \
172 vlib_set_next_frame_buffer (vm, node, next1, bi1); \
174 if (next_index == next2) \
181 vlib_set_next_frame_buffer (vm, node, next2, bi2); \
183 if (next_index == next3) \
191 vlib_set_next_frame_buffer (vm, node, next3, bi3); \
193 /* Change speculation: last 2 packets went to the same node*/ \
194 if (next2 == next3) \
196 vlib_put_next_frame (vm, node, next_index, n_left_to_next); \
197 next_index = next3; \
198 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
204 /** \brief Finish enqueueing one buffer forward in the graph.
205 Standard single loop boilerplate element. This is a MACRO,
206 with MULTIPLE SIDE EFFECTS. In the ideal case,
207 <code>next_index == next0</code>,
208 which means that the speculative enqueue at the top of the single loop
209 has correctly dealt with the packet in hand. In that case, the macro does
212 @param vm vlib_main_t pointer, varies by thread
213 @param node current node vlib_node_runtime_t pointer
214 @param next_index speculated next index used for both packets
215 @param to_next speculated vector pointer used for both packets
216 @param n_left_to_next number of slots left in speculated vector
217 @param bi0 first buffer index
218 @param next0 actual next index to be used for the first packet
220 @return @c next_index -- speculative next index to be used for future packets
221 @return @c to_next -- speculative frame to be used for future packets
222 @return @c n_left_to_next -- number of slots left in speculative frame
224 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
227 if (PREDICT_FALSE (next0 != next_index)) \
229 vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
230 next_index = next0; \
231 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
235 n_left_to_next -= 1; \
239 /** \brief Finish enqueueing one buffer forward in the graph, along with its
240 aux_data if possible. Standard single loop boilerplate element. This is a
241 MACRO, with MULTIPLE SIDE EFFECTS. In the ideal case, <code>next_index ==
242 next0</code>, which means that the speculative enqueue at the top of the
243 single loop has correctly dealt with the packet in hand. In that case, the
244 macro does nothing at all. This function MAY return to_next_aux = NULL if
245 next_index does not support aux data
247 @param vm vlib_main_t pointer, varies by thread
248 @param node current node vlib_node_runtime_t pointer
249 @param next_index speculated next index used for both packets
250 @param to_next speculated vector pointer used for both packets
251 @param to_next_aux speculated aux_data pointer used for both packets
252 @param n_left_to_next number of slots left in speculated vector
253 @param bi0 first buffer index
254 @param aux0 first aux_data
255 @param next0 actual next index to be used for the first packet
257 @return @c next_index -- speculative next index to be used for future packets
258 @return @c to_next -- speculative frame to be used for future packets
259 @return @c n_left_to_next -- number of slots left in speculative frame
261 #define vlib_validate_buffer_enqueue_with_aux_x1( \
262 vm, node, next_index, to_next, to_next_aux, n_left_to_next, bi0, aux0, \
267 if (PREDICT_FALSE (next0 != next_index)) \
269 vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1); \
270 next_index = next0; \
271 vlib_get_next_frame_with_aux_safe (vm, node, next_index, to_next, \
272 to_next_aux, n_left_to_next); \
278 to_next_aux[0] = aux0; \
281 n_left_to_next -= 1; \
287 generic_buffer_node_inline (vlib_main_t * vm,
288 vlib_node_runtime_t * node,
289 vlib_frame_t * frame,
293 void (*two_buffers) (vlib_main_t * vm,
298 u32 * next0, u32 * next1),
299 void (*one_buffer) (vlib_main_t * vm,
300 void *opaque1, uword opaque2,
304 u32 n_left_from, *from, *to_next;
307 from = vlib_frame_vector_args (frame);
308 n_left_from = frame->n_vectors;
309 next_index = node->cached_next_index;
311 if (node->flags & VLIB_NODE_FLAG_TRACE)
312 vlib_trace_frame_buffers_only (vm, node, from, frame->n_vectors,
313 /* stride */ 1, sizeof_trace);
315 while (n_left_from > 0)
319 vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
321 while (n_left_from >= 4 && n_left_to_next >= 2)
323 vlib_buffer_t *p0, *p1;
327 /* Prefetch next iteration. */
329 vlib_buffer_t *p2, *p3;
331 p2 = vlib_get_buffer (vm, from[2]);
332 p3 = vlib_get_buffer (vm, from[3]);
334 vlib_prefetch_buffer_header (p2, LOAD);
335 vlib_prefetch_buffer_header (p3, LOAD);
337 clib_prefetch_load (p2->data);
338 clib_prefetch_load (p3->data);
341 pi0 = to_next[0] = from[0];
342 pi1 = to_next[1] = from[1];
348 p0 = vlib_get_buffer (vm, pi0);
349 p1 = vlib_get_buffer (vm, pi1);
351 two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
353 vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
354 to_next, n_left_to_next,
355 pi0, pi1, next0, next1);
358 while (n_left_from > 0 && n_left_to_next > 0)
370 p0 = vlib_get_buffer (vm, pi0);
372 one_buffer (vm, opaque1, opaque2, p0, &next0);
374 vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
375 to_next, n_left_to_next,
379 vlib_put_next_frame (vm, node, next_index, n_left_to_next);
382 return frame->n_vectors;
385 /* Minimum size for the 'buffers' and 'nexts' arrays to be used when calling
386 * vlib_buffer_enqueue_to_next().
387 * Because of optimizations, vlib_buffer_enqueue_to_next() will access
388 * past 'count' elements in the 'buffers' and 'nexts' arrays, IOW it
390 * Those overflow elements are ignored in the final result so they do not
391 * need to be properly initialized, however if the array is allocated right
392 * before the end of a page and the next page is not mapped, accessing the
393 * overflow elements will trigger a segfault. */
394 #define VLIB_BUFFER_ENQUEUE_MIN_SIZE(n) round_pow2 ((n), 64)
396 static_always_inline void
397 vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
398 u32 * buffers, u16 * nexts, uword count)
400 vlib_buffer_enqueue_to_next_fn_t *fn;
401 fn = vlib_buffer_func_main.buffer_enqueue_to_next_fn;
402 (fn) (vm, node, buffers, nexts, count);
405 static_always_inline void
406 vlib_buffer_enqueue_to_next_with_aux (vlib_main_t *vm,
407 vlib_node_runtime_t *node, u32 *buffers,
408 u32 *aux_data, u16 *nexts, uword count)
410 vlib_buffer_enqueue_to_next_with_aux_fn_t *fn;
411 fn = vlib_buffer_func_main.buffer_enqueue_to_next_with_aux_fn;
412 (fn) (vm, node, buffers, aux_data, nexts, count);
415 static_always_inline void
416 vlib_buffer_enqueue_to_next_vec (vlib_main_t *vm, vlib_node_runtime_t *node,
417 u32 **buffers, u16 **nexts, uword count)
419 const u32 bl = vec_len (*buffers), nl = vec_len (*nexts);
420 const u32 c = VLIB_BUFFER_ENQUEUE_MIN_SIZE (count);
421 ASSERT (bl >= count && nl >= count);
422 vec_validate (*buffers, c);
423 vec_validate (*nexts, c);
424 vlib_buffer_enqueue_to_next (vm, node, *buffers, *nexts, count);
425 vec_set_len (*buffers, bl);
426 vec_set_len (*nexts, nl);
429 static_always_inline void
430 vlib_buffer_enqueue_to_single_next (vlib_main_t * vm,
431 vlib_node_runtime_t * node, u32 * buffers,
432 u16 next_index, u32 count)
434 vlib_buffer_enqueue_to_single_next_fn_t *fn;
435 fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_fn;
436 (fn) (vm, node, buffers, next_index, count);
439 static_always_inline void
440 vlib_buffer_enqueue_to_single_next_with_aux (vlib_main_t *vm,
441 vlib_node_runtime_t *node,
442 u32 *buffers, u32 *aux_data,
443 u16 next_index, u32 count)
445 vlib_buffer_enqueue_to_single_next_with_aux_fn_t *fn;
446 fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_with_aux_fn;
447 (fn) (vm, node, buffers, aux_data, next_index, count);
450 static_always_inline u32
451 vlib_buffer_enqueue_to_thread (vlib_main_t *vm, vlib_node_runtime_t *node,
452 u32 frame_queue_index, u32 *buffer_indices,
453 u16 *thread_indices, u32 n_packets,
454 int drop_on_congestion)
456 vlib_buffer_enqueue_to_thread_fn_t *fn;
457 fn = vlib_buffer_func_main.buffer_enqueue_to_thread_fn;
458 return (fn) (vm, node, frame_queue_index, buffer_indices, thread_indices,
459 n_packets, drop_on_congestion);
462 static_always_inline u32
463 vlib_buffer_enqueue_to_thread_with_aux (vlib_main_t *vm,
464 vlib_node_runtime_t *node,
465 u32 frame_queue_index,
466 u32 *buffer_indices, u32 *aux,
467 u16 *thread_indices, u32 n_packets,
468 int drop_on_congestion)
470 vlib_buffer_enqueue_to_thread_with_aux_fn_t *fn;
471 fn = vlib_buffer_func_main.buffer_enqueue_to_thread_with_aux_fn;
472 return (fn) (vm, node, frame_queue_index, buffer_indices, aux,
473 thread_indices, n_packets, drop_on_congestion);
476 #endif /* included_vlib_buffer_node_h */
479 * fd.io coding-style-patch-verification: ON
482 * eval: (c-set-style "gnu")