src/vlib/buffer_node.h

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * buffer_node.h: VLIB buffer handling node helper macros/inlines
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #ifndef included_vlib_buffer_node_h
  41 #define included_vlib_buffer_node_h
  42
  43 /** \file
  44     vlib buffer/node functions
  45 */
  46
  47 /** \brief Finish enqueueing two buffers forward in the graph.
  48  Standard dual loop boilerplate element. This is a MACRO,
  49  with MULTIPLE SIDE EFFECTS. In the ideal case,
  50  <code>next_index == next0 == next1</code>,
  51  which means that the speculative enqueue at the top of the dual loop
  52  has correctly dealt with both packets. In that case, the macro does
  53  nothing at all.
  54
  55  @param vm vlib_main_t pointer, varies by thread
  56  @param node current node vlib_node_runtime_t pointer
  57  @param next_index speculated next index used for both packets
  58  @param to_next speculated vector pointer used for both packets
  59  @param n_left_to_next number of slots left in speculated vector
  60  @param bi0 first buffer index
  61  @param bi1 second buffer index
  62  @param next0 actual next index to be used for the first packet
  63  @param next1 actual next index to be used for the second packet
  64
  65  @return @c next_index -- speculative next index to be used for future packets
  66  @return @c to_next -- speculative frame to be used for future packets
  67  @return @c n_left_to_next -- number of slots left in speculative frame
  68 */
  69
  70 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
  71 do {                                                                    \
  72   ASSERT (bi0 != 0);                                                    \
  73   ASSERT (bi1 != 0);                                                    \
  74   int enqueue_code = (next0 != next_index) + 2*(next1 != next_index);   \
  75                                                                         \
  76   if (PREDICT_FALSE (enqueue_code != 0))                                \
  77     {                                                                   \
  78       switch (enqueue_code)                                             \
  79         {                                                               \
  80         case 1:                                                         \
  81           /* A B A */                                                   \
  82           to_next[-2] = bi1;                                            \
  83           to_next -= 1;                                                 \
  84           n_left_to_next += 1;                                          \
  85           vlib_set_next_frame_buffer (vm, node, next0, bi0);            \
  86           break;                                                        \
  87                                                                         \
  88         case 2:                                                         \
  89           /* A A B */                                                   \
  90           to_next -= 1;                                                 \
  91           n_left_to_next += 1;                                          \
  92           vlib_set_next_frame_buffer (vm, node, next1, bi1);            \
  93           break;                                                        \
  94                                                                         \
  95         case 3:                                                         \
  96           /* A B B or A B C */                                          \
  97           to_next -= 2;                                                 \
  98           n_left_to_next += 2;                                          \
  99           vlib_set_next_frame_buffer (vm, node, next0, bi0);            \
 100           vlib_set_next_frame_buffer (vm, node, next1, bi1);            \
 101           if (next0 == next1)                                           \
 102             {                                                           \
 103               vlib_put_next_frame (vm, node, next_index,                \
 104                                    n_left_to_next);                     \
 105               next_index = next1;                                       \
 106               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
 107             }                                                           \
 108         }                                                               \
 109     }                                                                   \
 110 } while (0)
 111
 112
 113 /** \brief Finish enqueueing four buffers forward in the graph.
 114  Standard quad loop boilerplate element. This is a MACRO,
 115  with MULTIPLE SIDE EFFECTS. In the ideal case,
 116  <code>next_index == next0 == next1 == next2 == next3</code>,
 117  which means that the speculative enqueue at the top of the quad loop
 118  has correctly dealt with all four packets. In that case, the macro does
 119  nothing at all.
 120
 121  @param vm vlib_main_t pointer, varies by thread
 122  @param node current node vlib_node_runtime_t pointer
 123  @param next_index speculated next index used for both packets
 124  @param to_next speculated vector pointer used for both packets
 125  @param n_left_to_next number of slots left in speculated vector
 126  @param bi0 first buffer index
 127  @param bi1 second buffer index
 128  @param bi2 third buffer index
 129  @param bi3 fourth buffer index
 130  @param next0 actual next index to be used for the first packet
 131  @param next1 actual next index to be used for the second packet
 132  @param next2 actual next index to be used for the third packet
 133  @param next3 actual next index to be used for the fourth packet
 134
 135  @return @c next_index -- speculative next index to be used for future packets
 136  @return @c to_next -- speculative frame to be used for future packets
 137  @return @c n_left_to_next -- number of slots left in speculative frame
 138 */
 139
 140 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
 141 do {                                                                    \
 142   ASSERT (bi0 != 0);                                                    \
 143   ASSERT (bi1 != 0);                                                    \
 144   ASSERT (bi2 != 0);                                                    \
 145   ASSERT (bi3 != 0);                                                    \
 146   /* After the fact: check the [speculative] enqueue to "next" */       \
 147   u32 fix_speculation = (next_index ^ next0) | (next_index ^ next1)     \
 148     | (next_index ^ next2) | (next_index ^ next3);                      \
 149   if (PREDICT_FALSE(fix_speculation))                                   \
 150     {                                                                   \
 151       /* rewind... */                                                   \
 152       to_next -= 4;                                                     \
 153       n_left_to_next += 4;                                              \
 154                                                                         \
 155       /* If bi0 belongs to "next", send it there */                     \
 156       if (next_index == next0)                                          \
 157         {                                                               \
 158           to_next[0] = bi0;                                             \
 159           to_next++;                                                    \
 160           n_left_to_next --;                                            \
 161         }                                                               \
 162       else              /* send it where it needs to go */              \
 163         vlib_set_next_frame_buffer (vm, node, next0, bi0);              \
 164                                                                         \
 165       if (next_index == next1)                                          \
 166         {                                                               \
 167           to_next[0] = bi1;                                             \
 168           to_next++;                                                    \
 169           n_left_to_next --;                                            \
 170         }                                                               \
 171       else                                                              \
 172         vlib_set_next_frame_buffer (vm, node, next1, bi1);              \
 173                                                                         \
 174       if (next_index == next2)                                          \
 175         {                                                               \
 176           to_next[0] = bi2;                                             \
 177           to_next++;                                                    \
 178           n_left_to_next --;                                            \
 179         }                                                               \
 180       else                                                              \
 181         vlib_set_next_frame_buffer (vm, node, next2, bi2);              \
 182                                                                         \
 183       if (next_index == next3)                                          \
 184         {                                                               \
 185           to_next[0] = bi3;                                             \
 186           to_next++;                                                    \
 187           n_left_to_next --;                                            \
 188         }                                                               \
 189       else                                                              \
 190         {                                                               \
 191           vlib_set_next_frame_buffer (vm, node, next3, bi3);            \
 192                                                                         \
 193           /* Change speculation: last 2 packets went to the same node*/ \
 194           if (next2 == next3)                                           \
 195             {                                                           \
 196               vlib_put_next_frame (vm, node, next_index, n_left_to_next); \
 197               next_index = next3;                                       \
 198               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
 199             }                                                           \
 200         }                                                               \
 201     }                                                                   \
 202  } while(0);
 203
 204 /** \brief Finish enqueueing one buffer forward in the graph.
 205  Standard single loop boilerplate element. This is a MACRO,
 206  with MULTIPLE SIDE EFFECTS. In the ideal case,
 207  <code>next_index == next0</code>,
 208  which means that the speculative enqueue at the top of the single loop
 209  has correctly dealt with the packet in hand. In that case, the macro does
 210  nothing at all.
 211
 212  @param vm vlib_main_t pointer, varies by thread
 213  @param node current node vlib_node_runtime_t pointer
 214  @param next_index speculated next index used for both packets
 215  @param to_next speculated vector pointer used for both packets
 216  @param n_left_to_next number of slots left in speculated vector
 217  @param bi0 first buffer index
 218  @param next0 actual next index to be used for the first packet
 219
 220  @return @c next_index -- speculative next index to be used for future packets
 221  @return @c to_next -- speculative frame to be used for future packets
 222  @return @c n_left_to_next -- number of slots left in speculative frame
 223 */
 224 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
 225 do {                                                                    \
 226   ASSERT (bi0 != 0);                                                    \
 227   if (PREDICT_FALSE (next0 != next_index))                              \
 228     {                                                                   \
 229       vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1);   \
 230       next_index = next0;                                               \
 231       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
 232                                                                         \
 233       to_next[0] = bi0;                                                 \
 234       to_next += 1;                                                     \
 235       n_left_to_next -= 1;                                              \
 236     }                                                                   \
 237 } while (0)
 238
 239 /** \brief Finish enqueueing one buffer forward in the graph, along with its
 240  aux_data if possible. Standard single loop boilerplate element. This is a
 241  MACRO, with MULTIPLE SIDE EFFECTS. In the ideal case, <code>next_index ==
 242  next0</code>, which means that the speculative enqueue at the top of the
 243  single loop has correctly dealt with the packet in hand. In that case, the
 244  macro does nothing at all. This function MAY return to_next_aux = NULL if
 245  next_index does not support aux data
 246
 247  @param vm vlib_main_t pointer, varies by thread
 248  @param node current node vlib_node_runtime_t pointer
 249  @param next_index speculated next index used for both packets
 250  @param to_next speculated vector pointer used for both packets
 251  @param to_next_aux speculated aux_data pointer used for both packets
 252  @param n_left_to_next number of slots left in speculated vector
 253  @param bi0 first buffer index
 254  @param aux0 first aux_data
 255  @param next0 actual next index to be used for the first packet
 256
 257  @return @c next_index -- speculative next index to be used for future packets
 258  @return @c to_next -- speculative frame to be used for future packets
 259  @return @c n_left_to_next -- number of slots left in speculative frame
 260 */
 261 #define vlib_validate_buffer_enqueue_with_aux_x1(                             \
 262   vm, node, next_index, to_next, to_next_aux, n_left_to_next, bi0, aux0,      \
 263   next0)                                                                      \
 264   do                                                                          \
 265     {                                                                         \
 266       ASSERT (bi0 != 0);                                                      \
 267       if (PREDICT_FALSE (next0 != next_index))                                \
 268         {                                                                     \
 269           vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1);     \
 270           next_index = next0;                                                 \
 271           vlib_get_next_frame_with_aux_safe (vm, node, next_index, to_next,   \
 272                                              to_next_aux, n_left_to_next);    \
 273                                                                               \
 274           to_next[0] = bi0;                                                   \
 275           to_next += 1;                                                       \
 276           if (to_next_aux)                                                    \
 277             {                                                                 \
 278               to_next_aux[0] = aux0;                                          \
 279               to_next_aux += 1;                                               \
 280             }                                                                 \
 281           n_left_to_next -= 1;                                                \
 282         }                                                                     \
 283     }                                                                         \
 284   while (0)
 285
 286 always_inline uword
 287 generic_buffer_node_inline (vlib_main_t * vm,
 288                             vlib_node_runtime_t * node,
 289                             vlib_frame_t * frame,
 290                             uword sizeof_trace,
 291                             void *opaque1,
 292                             uword opaque2,
 293                             void (*two_buffers) (vlib_main_t * vm,
 294                                                  void *opaque1,
 295                                                  uword opaque2,
 296                                                  vlib_buffer_t * b0,
 297                                                  vlib_buffer_t * b1,
 298                                                  u32 * next0, u32 * next1),
 299                             void (*one_buffer) (vlib_main_t * vm,
 300                                                 void *opaque1, uword opaque2,
 301                                                 vlib_buffer_t * b0,
 302                                                 u32 * next0))
 303 {
 304   u32 n_left_from, *from, *to_next;
 305   u32 next_index;
 306
 307   from = vlib_frame_vector_args (frame);
 308   n_left_from = frame->n_vectors;
 309   next_index = node->cached_next_index;
 310
 311   if (node->flags & VLIB_NODE_FLAG_TRACE)
 312     vlib_trace_frame_buffers_only (vm, node, from, frame->n_vectors,
 313                                    /* stride */ 1, sizeof_trace);
 314
 315   while (n_left_from > 0)
 316     {
 317       u32 n_left_to_next;
 318
 319       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 320
 321       while (n_left_from >= 4 && n_left_to_next >= 2)
 322         {
 323           vlib_buffer_t *p0, *p1;
 324           u32 pi0, next0;
 325           u32 pi1, next1;
 326
 327           /* Prefetch next iteration. */
 328           {
 329             vlib_buffer_t *p2, *p3;
 330
 331             p2 = vlib_get_buffer (vm, from[2]);
 332             p3 = vlib_get_buffer (vm, from[3]);
 333
 334             vlib_prefetch_buffer_header (p2, LOAD);
 335             vlib_prefetch_buffer_header (p3, LOAD);
 336
 337             clib_prefetch_load (p2->data);
 338             clib_prefetch_load (p3->data);
 339           }
 340
 341           pi0 = to_next[0] = from[0];
 342           pi1 = to_next[1] = from[1];
 343           from += 2;
 344           to_next += 2;
 345           n_left_from -= 2;
 346           n_left_to_next -= 2;
 347
 348           p0 = vlib_get_buffer (vm, pi0);
 349           p1 = vlib_get_buffer (vm, pi1);
 350
 351           two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
 352
 353           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 354                                            to_next, n_left_to_next,
 355                                            pi0, pi1, next0, next1);
 356         }
 357
 358       while (n_left_from > 0 && n_left_to_next > 0)
 359         {
 360           vlib_buffer_t *p0;
 361           u32 pi0, next0;
 362
 363           pi0 = from[0];
 364           to_next[0] = pi0;
 365           from += 1;
 366           to_next += 1;
 367           n_left_from -= 1;
 368           n_left_to_next -= 1;
 369
 370           p0 = vlib_get_buffer (vm, pi0);
 371
 372           one_buffer (vm, opaque1, opaque2, p0, &next0);
 373
 374           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 375                                            to_next, n_left_to_next,
 376                                            pi0, next0);
 377         }
 378
 379       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 380     }
 381
 382   return frame->n_vectors;
 383 }
 384
 385 /* Minimum size for the 'buffers' and 'nexts' arrays to be used when calling
 386  * vlib_buffer_enqueue_to_next().
 387  * Because of optimizations, vlib_buffer_enqueue_to_next() will access
 388  * past 'count' elements in the 'buffers' and 'nexts' arrays, IOW it
 389  * will overflow.
 390  * Those overflow elements are ignored in the final result so they do not
 391  * need to be properly initialized, however if the array is allocated right
 392  * before the end of a page and the next page is not mapped, accessing the
 393  * overflow elements will trigger a segfault. */
 394 #define VLIB_BUFFER_ENQUEUE_MIN_SIZE(n) round_pow2 ((n), 64)
 395
 396 static_always_inline void
 397 vlib_buffer_enqueue_to_next (vlib_main_t * vm, vlib_node_runtime_t * node,
 398                              u32 * buffers, u16 * nexts, uword count)
 399 {
 400   vlib_buffer_enqueue_to_next_fn_t *fn;
 401   fn = vlib_buffer_func_main.buffer_enqueue_to_next_fn;
 402   (fn) (vm, node, buffers, nexts, count);
 403 }
 404
 405 static_always_inline void
 406 vlib_buffer_enqueue_to_next_with_aux (vlib_main_t *vm,
 407                                       vlib_node_runtime_t *node, u32 *buffers,
 408                                       u32 *aux_data, u16 *nexts, uword count)
 409 {
 410   vlib_buffer_enqueue_to_next_with_aux_fn_t *fn;
 411   fn = vlib_buffer_func_main.buffer_enqueue_to_next_with_aux_fn;
 412   (fn) (vm, node, buffers, aux_data, nexts, count);
 413 }
 414
 415 static_always_inline void
 416 vlib_buffer_enqueue_to_next_vec (vlib_main_t *vm, vlib_node_runtime_t *node,
 417                                  u32 **buffers, u16 **nexts, uword count)
 418 {
 419   const u32 bl = vec_len (*buffers), nl = vec_len (*nexts);
 420   const u32 c = VLIB_BUFFER_ENQUEUE_MIN_SIZE (count);
 421   ASSERT (bl >= count && nl >= count);
 422   vec_validate (*buffers, c);
 423   vec_validate (*nexts, c);
 424   vlib_buffer_enqueue_to_next (vm, node, *buffers, *nexts, count);
 425   vec_set_len (*buffers, bl);
 426   vec_set_len (*nexts, nl);
 427 }
 428
 429 static_always_inline void
 430 vlib_buffer_enqueue_to_single_next (vlib_main_t * vm,
 431                                     vlib_node_runtime_t * node, u32 * buffers,
 432                                     u16 next_index, u32 count)
 433 {
 434   vlib_buffer_enqueue_to_single_next_fn_t *fn;
 435   fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_fn;
 436   (fn) (vm, node, buffers, next_index, count);
 437 }
 438
 439 static_always_inline void
 440 vlib_buffer_enqueue_to_single_next_with_aux (vlib_main_t *vm,
 441                                              vlib_node_runtime_t *node,
 442                                              u32 *buffers, u32 *aux_data,
 443                                              u16 next_index, u32 count)
 444 {
 445   vlib_buffer_enqueue_to_single_next_with_aux_fn_t *fn;
 446   fn = vlib_buffer_func_main.buffer_enqueue_to_single_next_with_aux_fn;
 447   (fn) (vm, node, buffers, aux_data, next_index, count);
 448 }
 449
 450 static_always_inline u32
 451 vlib_buffer_enqueue_to_thread (vlib_main_t *vm, vlib_node_runtime_t *node,
 452                                u32 frame_queue_index, u32 *buffer_indices,
 453                                u16 *thread_indices, u32 n_packets,
 454                                int drop_on_congestion)
 455 {
 456   vlib_buffer_enqueue_to_thread_fn_t *fn;
 457   fn = vlib_buffer_func_main.buffer_enqueue_to_thread_fn;
 458   return (fn) (vm, node, frame_queue_index, buffer_indices, thread_indices,
 459                n_packets, drop_on_congestion);
 460 }
 461
 462 static_always_inline u32
 463 vlib_buffer_enqueue_to_thread_with_aux (vlib_main_t *vm,
 464                                         vlib_node_runtime_t *node,
 465                                         u32 frame_queue_index,
 466                                         u32 *buffer_indices, u32 *aux,
 467                                         u16 *thread_indices, u32 n_packets,
 468                                         int drop_on_congestion)
 469 {
 470   vlib_buffer_enqueue_to_thread_with_aux_fn_t *fn;
 471   fn = vlib_buffer_func_main.buffer_enqueue_to_thread_with_aux_fn;
 472   return (fn) (vm, node, frame_queue_index, buffer_indices, aux,
 473                thread_indices, n_packets, drop_on_congestion);
 474 }
 475
 476 #endif /* included_vlib_buffer_node_h */
 477
 478 /*
 479  * fd.io coding-style-patch-verification: ON
 480  *
 481  * Local Variables:
 482  * eval: (c-set-style "gnu")
 483  * End:
 484  */