vlib/vlib/buffer_node.h

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 /*
  16  * buffer_node.h: VLIB buffer handling node helper macros/inlines
  17  *
  18  * Copyright (c) 2008 Eliot Dresselhaus
  19  *
  20  * Permission is hereby granted, free of charge, to any person obtaining
  21  * a copy of this software and associated documentation files (the
  22  * "Software"), to deal in the Software without restriction, including
  23  * without limitation the rights to use, copy, modify, merge, publish,
  24  * distribute, sublicense, and/or sell copies of the Software, and to
  25  * permit persons to whom the Software is furnished to do so, subject to
  26  * the following conditions:
  27  *
  28  * The above copyright notice and this permission notice shall be
  29  * included in all copies or substantial portions of the Software.
  30  *
  31  *  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  32  *  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  33  *  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  34  *  NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  35  *  LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  36  *  OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  37  *  WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  38  */
  39
  40 #ifndef included_vlib_buffer_node_h
  41 #define included_vlib_buffer_node_h
  42
  43 /** \file
  44     vlib buffer/node functions
  45 */
  46
  47 /** \brief Finish enqueueing two buffers forward in the graph.
  48  Standard dual loop boilerplate element. This is a MACRO,
  49  with MULTIPLE SIDE EFFECTS. In the ideal case,
  50  <code>next_index == next0 == next1</code>,
  51  which means that the speculative enqueue at the top of the dual loop
  52  has correctly dealt with both packets. In that case, the macro does
  53  nothing at all.
  54
  55  @param vm vlib_main_t pointer, varies by thread
  56  @param node current node vlib_node_runtime_t pointer
  57  @param next_index speculated next index used for both packets
  58  @param to_next speculated vector pointer used for both packets
  59  @param n_left_to_next number of slots left in speculated vector
  60  @param bi0 first buffer index
  61  @param bi1 second buffer index
  62  @param next0 actual next index to be used for the first packet
  63  @param next1 actual next index to be used for the second packet
  64
  65  @return @c next_index -- speculative next index to be used for future packets
  66  @return @c to_next -- speculative frame to be used for future packets
  67  @return @c n_left_to_next -- number of slots left in speculative frame
  68 */
  69
  70 #define vlib_validate_buffer_enqueue_x2(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,next0,next1) \
  71 do {                                                                    \
  72   int enqueue_code = (next0 != next_index) + 2*(next1 != next_index);   \
  73                                                                         \
  74   if (PREDICT_FALSE (enqueue_code != 0))                                \
  75     {                                                                   \
  76       switch (enqueue_code)                                             \
  77         {                                                               \
  78         case 1:                                                         \
  79           /* A B A */                                                   \
  80           to_next[-2] = bi1;                                            \
  81           to_next -= 1;                                                 \
  82           n_left_to_next += 1;                                          \
  83           vlib_set_next_frame_buffer (vm, node, next0, bi0);            \
  84           break;                                                        \
  85                                                                         \
  86         case 2:                                                         \
  87           /* A A B */                                                   \
  88           to_next -= 1;                                                 \
  89           n_left_to_next += 1;                                          \
  90           vlib_set_next_frame_buffer (vm, node, next1, bi1);            \
  91           break;                                                        \
  92                                                                         \
  93         case 3:                                                         \
  94           /* A B B or A B C */                                          \
  95           to_next -= 2;                                                 \
  96           n_left_to_next += 2;                                          \
  97           vlib_set_next_frame_buffer (vm, node, next0, bi0);            \
  98           vlib_set_next_frame_buffer (vm, node, next1, bi1);            \
  99           if (next0 == next1)                                           \
 100             {                                                           \
 101               vlib_put_next_frame (vm, node, next_index,                \
 102                                    n_left_to_next);                     \
 103               next_index = next1;                                       \
 104               vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
 105             }                                                           \
 106         }                                                               \
 107     }                                                                   \
 108 } while (0)
 109
 110
 111 /** \brief Finish enqueueing four buffers forward in the graph.
 112  Standard quad loop boilerplate element. This is a MACRO,
 113  with MULTIPLE SIDE EFFECTS. In the ideal case,
 114  <code>next_index == next0 == next1 == next2 == next3</code>,
 115  which means that the speculative enqueue at the top of the quad loop
 116  has correctly dealt with all four packets. In that case, the macro does
 117  nothing at all.
 118
 119  @param vm vlib_main_t pointer, varies by thread
 120  @param node current node vlib_node_runtime_t pointer
 121  @param next_index speculated next index used for both packets
 122  @param to_next speculated vector pointer used for both packets
 123  @param n_left_to_next number of slots left in speculated vector
 124  @param bi0 first buffer index
 125  @param bi1 second buffer index
 126  @param bi2 third buffer index
 127  @param bi3 fourth buffer index
 128  @param next0 actual next index to be used for the first packet
 129  @param next1 actual next index to be used for the second packet
 130  @param next2 actual next index to be used for the third packet
 131  @param next3 actual next index to be used for the fourth packet
 132
 133  @return @c next_index -- speculative next index to be used for future packets
 134  @return @c to_next -- speculative frame to be used for future packets
 135  @return @c n_left_to_next -- number of slots left in speculative frame
 136 */
 137
 138 #define vlib_validate_buffer_enqueue_x4(vm,node,next_index,to_next,n_left_to_next,bi0,bi1,bi2,bi3,next0,next1,next2,next3) \
 139 do {                                                                    \
 140   /* After the fact: check the [speculative] enqueue to "next" */       \
 141   u32 fix_speculation = next_index != next0 || next_index != next1      \
 142     || next_index != next2 || next_index != next3;                      \
 143   if (PREDICT_FALSE(fix_speculation))                                   \
 144     {                                                                   \
 145       /* rewind... */                                                   \
 146       to_next -= 4;                                                     \
 147       n_left_to_next += 4;                                              \
 148                                                                         \
 149       /* If bi0 belongs to "next", send it there */                     \
 150       if (next_index == next0)                                          \
 151         {                                                               \
 152           to_next[0] = bi0;                                             \
 153           to_next++;                                                    \
 154           n_left_to_next --;                                            \
 155         }                                                               \
 156       else              /* send it where it needs to go */              \
 157         vlib_set_next_frame_buffer (vm, node, next0, bi0);              \
 158                                                                         \
 159       if (next_index == next1)                                          \
 160         {                                                               \
 161           to_next[0] = bi1;                                             \
 162           to_next++;                                                    \
 163           n_left_to_next --;                                            \
 164         }                                                               \
 165       else                                                              \
 166         vlib_set_next_frame_buffer (vm, node, next1, bi1);              \
 167                                                                         \
 168       if (next_index == next2)                                          \
 169         {                                                               \
 170           to_next[0] = bi2;                                             \
 171           to_next++;                                                    \
 172           n_left_to_next --;                                            \
 173         }                                                               \
 174       else                                                              \
 175         vlib_set_next_frame_buffer (vm, node, next2, bi2);              \
 176                                                                         \
 177       if (next_index == next3)                                          \
 178         {                                                               \
 179           to_next[0] = bi3;                                             \
 180           to_next++;                                                    \
 181           n_left_to_next --;                                            \
 182         }                                                               \
 183       else                                                              \
 184         vlib_set_next_frame_buffer (vm, node, next3, bi3);              \
 185                                                                         \
 186       /* Change speculation: last 2 packets went to the same node */    \
 187       if (next2 == next3)                                               \
 188         {                                                               \
 189           vlib_put_next_frame (vm, node, next_index, n_left_to_next);   \
 190           next_index = next3;                                           \
 191           vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
 192         }                                                               \
 193     }                                                                   \
 194  } while(0);
 195
 196 /** \brief Finish enqueueing one buffer forward in the graph.
 197  Standard single loop boilerplate element. This is a MACRO,
 198  with MULTIPLE SIDE EFFECTS. In the ideal case,
 199  <code>next_index == next0</code>,
 200  which means that the speculative enqueue at the top of the single loop
 201  has correctly dealt with the packet in hand. In that case, the macro does
 202  nothing at all.
 203
 204  @param vm vlib_main_t pointer, varies by thread
 205  @param node current node vlib_node_runtime_t pointer
 206  @param next_index speculated next index used for both packets
 207  @param to_next speculated vector pointer used for both packets
 208  @param n_left_to_next number of slots left in speculated vector
 209  @param bi0 first buffer index
 210  @param next0 actual next index to be used for the first packet
 211
 212  @return @c next_index -- speculative next index to be used for future packets
 213  @return @c to_next -- speculative frame to be used for future packets
 214  @return @c n_left_to_next -- number of slots left in speculative frame
 215 */
 216 #define vlib_validate_buffer_enqueue_x1(vm,node,next_index,to_next,n_left_to_next,bi0,next0) \
 217 do {                                                                    \
 218   if (PREDICT_FALSE (next0 != next_index))                              \
 219     {                                                                   \
 220       vlib_put_next_frame (vm, node, next_index, n_left_to_next + 1);   \
 221       next_index = next0;                                               \
 222       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next); \
 223                                                                         \
 224       to_next[0] = bi0;                                                 \
 225       to_next += 1;                                                     \
 226       n_left_to_next -= 1;                                              \
 227     }                                                                   \
 228 } while (0)
 229
 230 always_inline uword
 231 generic_buffer_node_inline (vlib_main_t * vm,
 232                             vlib_node_runtime_t * node,
 233                             vlib_frame_t * frame,
 234                             uword sizeof_trace,
 235                             void *opaque1,
 236                             uword opaque2,
 237                             void (*two_buffers) (vlib_main_t * vm,
 238                                                  void *opaque1,
 239                                                  uword opaque2,
 240                                                  vlib_buffer_t * b0,
 241                                                  vlib_buffer_t * b1,
 242                                                  u32 * next0, u32 * next1),
 243                             void (*one_buffer) (vlib_main_t * vm,
 244                                                 void *opaque1, uword opaque2,
 245                                                 vlib_buffer_t * b0,
 246                                                 u32 * next0))
 247 {
 248   u32 n_left_from, *from, *to_next;
 249   u32 next_index;
 250
 251   from = vlib_frame_vector_args (frame);
 252   n_left_from = frame->n_vectors;
 253   next_index = node->cached_next_index;
 254
 255   if (node->flags & VLIB_NODE_FLAG_TRACE)
 256     vlib_trace_frame_buffers_only (vm, node, from, frame->n_vectors,
 257                                    /* stride */ 1, sizeof_trace);
 258
 259   while (n_left_from > 0)
 260     {
 261       u32 n_left_to_next;
 262
 263       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 264
 265       while (n_left_from >= 4 && n_left_to_next >= 2)
 266         {
 267           vlib_buffer_t *p0, *p1;
 268           u32 pi0, next0;
 269           u32 pi1, next1;
 270
 271           /* Prefetch next iteration. */
 272           {
 273             vlib_buffer_t *p2, *p3;
 274
 275             p2 = vlib_get_buffer (vm, from[2]);
 276             p3 = vlib_get_buffer (vm, from[3]);
 277
 278             vlib_prefetch_buffer_header (p2, LOAD);
 279             vlib_prefetch_buffer_header (p3, LOAD);
 280
 281             CLIB_PREFETCH (p2->data, 64, LOAD);
 282             CLIB_PREFETCH (p3->data, 64, LOAD);
 283           }
 284
 285           pi0 = to_next[0] = from[0];
 286           pi1 = to_next[1] = from[1];
 287           from += 2;
 288           to_next += 2;
 289           n_left_from -= 2;
 290           n_left_to_next -= 2;
 291
 292           p0 = vlib_get_buffer (vm, pi0);
 293           p1 = vlib_get_buffer (vm, pi1);
 294
 295           two_buffers (vm, opaque1, opaque2, p0, p1, &next0, &next1);
 296
 297           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 298                                            to_next, n_left_to_next,
 299                                            pi0, pi1, next0, next1);
 300         }
 301
 302       while (n_left_from > 0 && n_left_to_next > 0)
 303         {
 304           vlib_buffer_t *p0;
 305           u32 pi0, next0;
 306
 307           pi0 = from[0];
 308           to_next[0] = pi0;
 309           from += 1;
 310           to_next += 1;
 311           n_left_from -= 1;
 312           n_left_to_next -= 1;
 313
 314           p0 = vlib_get_buffer (vm, pi0);
 315
 316           one_buffer (vm, opaque1, opaque2, p0, &next0);
 317
 318           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 319                                            to_next, n_left_to_next,
 320                                            pi0, next0);
 321         }
 322
 323       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 324     }
 325
 326   return frame->n_vectors;
 327 }
 328
 329 #endif /* included_vlib_buffer_node_h */
 330
 331 /*
 332  * fd.io coding-style-patch-verification: ON
 333  *
 334  * Local Variables:
 335  * eval: (c-set-style "gnu")
 336  * End:
 337  */