src/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/policer/police_inlines.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/classify/policer_classify.h>
  24 #include <vnet/classify/vnet_classify.h>
  25 #include <vnet/l2/feat_bitmap.h>
  26 #include <vnet/l2/l2_input.h>
  27
  28
  29 /* Dispatch functions meant to be instantiated elsewhere */
  30
  31 typedef struct
  32 {
  33   u32 next_index;
  34   u32 sw_if_index;
  35   u32 policer_index;
  36 } vnet_policer_trace_t;
  37
  38 /* packet trace format function */
  39 static u8 *
  40 format_policer_trace (u8 * s, va_list * args)
  41 {
  42   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  43   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  44   vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
  45
  46   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  47               t->sw_if_index, t->policer_index, t->next_index);
  48   return s;
  49 }
  50
  51 #define foreach_vnet_policer_error              \
  52 _(TRANSMIT, "Packets Transmitted")              \
  53 _(DROP, "Packets Dropped")
  54
  55 typedef enum
  56 {
  57 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  58   foreach_vnet_policer_error
  59 #undef _
  60     VNET_POLICER_N_ERROR,
  61 } vnet_policer_error_t;
  62
  63 static char *vnet_policer_error_strings[] = {
  64 #define _(sym,string) string,
  65   foreach_vnet_policer_error
  66 #undef _
  67 };
  68
  69 static inline uword
  70 vnet_policer_inline (vlib_main_t * vm,
  71                      vlib_node_runtime_t * node,
  72                      vlib_frame_t * frame, vnet_policer_index_t which)
  73 {
  74   u32 n_left_from, *from, *to_next;
  75   vnet_policer_next_t next_index;
  76   vnet_policer_main_t *pm = &vnet_policer_main;
  77   u64 time_in_policer_periods;
  78   u32 transmitted = 0;
  79
  80   time_in_policer_periods =
  81     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
  82
  83   from = vlib_frame_vector_args (frame);
  84   n_left_from = frame->n_vectors;
  85   next_index = node->cached_next_index;
  86
  87   while (n_left_from > 0)
  88     {
  89       u32 n_left_to_next;
  90
  91       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  92
  93       while (n_left_from >= 4 && n_left_to_next >= 2)
  94         {
  95           u32 bi0, bi1;
  96           vlib_buffer_t *b0, *b1;
  97           u32 next0, next1;
  98           u32 sw_if_index0, sw_if_index1;
  99           u32 pi0 = 0, pi1 = 0;
 100           u8 act0, act1;
 101
 102           /* Prefetch next iteration. */
 103           {
 104             vlib_buffer_t *b2, *b3;
 105
 106             b2 = vlib_get_buffer (vm, from[2]);
 107             b3 = vlib_get_buffer (vm, from[3]);
 108
 109             vlib_prefetch_buffer_header (b2, LOAD);
 110             vlib_prefetch_buffer_header (b3, LOAD);
 111           }
 112
 113           /* speculatively enqueue b0 and b1 to the current next frame */
 114           to_next[0] = bi0 = from[0];
 115           to_next[1] = bi1 = from[1];
 116           from += 2;
 117           to_next += 2;
 118           n_left_from -= 2;
 119           n_left_to_next -= 2;
 120
 121           b0 = vlib_get_buffer (vm, bi0);
 122           b1 = vlib_get_buffer (vm, bi1);
 123
 124           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 125           next0 = VNET_POLICER_NEXT_TRANSMIT;
 126
 127           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 128           next1 = VNET_POLICER_NEXT_TRANSMIT;
 129
 130
 131           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 132             {
 133               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 134               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 135             }
 136
 137           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 138             {
 139               pi0 = vnet_buffer (b0)->policer.index;
 140               pi1 = vnet_buffer (b1)->policer.index;
 141             }
 142
 143           if (which == VNET_POLICER_INDEX_BY_EITHER)
 144             {
 145               pi0 = vnet_buffer (b0)->policer.index;
 146               pi0 = (pi0 != ~0) ? pi0 :
 147                 pm->policer_index_by_sw_if_index[sw_if_index0];
 148               pi1 = vnet_buffer (b1)->policer.index;
 149               pi1 = (pi1 != ~0) ? pi1 :
 150                 pm->policer_index_by_sw_if_index[sw_if_index1];
 151             }
 152
 153           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 154                                       POLICE_CONFORM /* no chaining */ );
 155
 156           act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
 157                                       POLICE_CONFORM /* no chaining */ );
 158
 159           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 160             {
 161               next0 = VNET_POLICER_NEXT_DROP;
 162               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 163             }
 164           else                  /* transmit or mark-and-transmit action */
 165             {
 166               transmitted++;
 167             }
 168
 169           if (PREDICT_FALSE (act1 == SSE2_QOS_ACTION_DROP))     /* drop action */
 170             {
 171               next1 = VNET_POLICER_NEXT_DROP;
 172               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 173             }
 174           else                  /* transmit or mark-and-transmit action */
 175             {
 176               transmitted++;
 177             }
 178
 179
 180           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
 181             {
 182               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 183                 {
 184                   vnet_policer_trace_t *t =
 185                     vlib_add_trace (vm, node, b0, sizeof (*t));
 186                   t->sw_if_index = sw_if_index0;
 187                   t->next_index = next0;
 188                 }
 189               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 190                 {
 191                   vnet_policer_trace_t *t =
 192                     vlib_add_trace (vm, node, b1, sizeof (*t));
 193                   t->sw_if_index = sw_if_index1;
 194                   t->next_index = next1;
 195                 }
 196             }
 197
 198           /* verify speculative enqueues, maybe switch current next frame */
 199           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 200                                            to_next, n_left_to_next,
 201                                            bi0, bi1, next0, next1);
 202         }
 203
 204       while (n_left_from > 0 && n_left_to_next > 0)
 205         {
 206           u32 bi0;
 207           vlib_buffer_t *b0;
 208           u32 next0;
 209           u32 sw_if_index0;
 210           u32 pi0 = 0;
 211           u8 act0;
 212
 213           bi0 = from[0];
 214           to_next[0] = bi0;
 215           from += 1;
 216           to_next += 1;
 217           n_left_from -= 1;
 218           n_left_to_next -= 1;
 219
 220           b0 = vlib_get_buffer (vm, bi0);
 221
 222           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 223           next0 = VNET_POLICER_NEXT_TRANSMIT;
 224
 225           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 226             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 227
 228           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 229             pi0 = vnet_buffer (b0)->policer.index;
 230
 231           if (which == VNET_POLICER_INDEX_BY_EITHER)
 232             {
 233               pi0 = vnet_buffer (b0)->policer.index;
 234               pi0 = (pi0 != ~0) ? pi0 :
 235                 pm->policer_index_by_sw_if_index[sw_if_index0];
 236             }
 237
 238           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 239                                       POLICE_CONFORM /* no chaining */ );
 240
 241           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 242             {
 243               next0 = VNET_POLICER_NEXT_DROP;
 244               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 245             }
 246           else                  /* transmit or mark-and-transmit action */
 247             {
 248               transmitted++;
 249             }
 250
 251           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 252                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 253             {
 254               vnet_policer_trace_t *t =
 255                 vlib_add_trace (vm, node, b0, sizeof (*t));
 256               t->sw_if_index = sw_if_index0;
 257               t->next_index = next0;
 258               t->policer_index = pi0;
 259             }
 260
 261           /* verify speculative enqueue, maybe switch current next frame */
 262           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 263                                            to_next, n_left_to_next,
 264                                            bi0, next0);
 265         }
 266
 267       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 268     }
 269
 270   vlib_node_increment_counter (vm, node->node_index,
 271                                VNET_POLICER_ERROR_TRANSMIT, transmitted);
 272   return frame->n_vectors;
 273 }
 274
 275 VLIB_NODE_FN (policer_by_sw_if_index_node) (vlib_main_t * vm,
 276                                             vlib_node_runtime_t * node,
 277                                             vlib_frame_t * frame)
 278 {
 279   return vnet_policer_inline (vm, node, frame,
 280                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 281 }
 282
 283 #ifndef CLIB_MARCH_VARIANT
 284 uword
 285 vnet_policer_by_opaque (vlib_main_t * vm,
 286                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 287 {
 288   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_OPAQUE);
 289 }
 290
 291 uword
 292 vnet_policer_by_either (vlib_main_t * vm,
 293                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 294 {
 295   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_EITHER);
 296 }
 297
 298 void
 299 vnet_policer_node_funcs_reference (void)
 300 {
 301 }
 302 #endif /* CLIB_MARCH_VARIANT */
 303
 304
 305 #define TEST_CODE 1
 306
 307 #ifdef TEST_CODE
 308
 309 /* *INDENT-OFF* */
 310 VLIB_REGISTER_NODE (policer_by_sw_if_index_node) = {
 311   .name = "policer-by-sw-if-index",
 312   .vector_size = sizeof (u32),
 313   .format_trace = format_policer_trace,
 314   .type = VLIB_NODE_TYPE_INTERNAL,
 315
 316   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 317   .error_strings = vnet_policer_error_strings,
 318
 319   .n_next_nodes = VNET_POLICER_N_NEXT,
 320
 321   /* edit / add dispositions here */
 322   .next_nodes = {
 323     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 324     [VNET_POLICER_NEXT_DROP] = "error-drop",
 325   },
 326 };
 327 /* *INDENT-ON* */
 328
 329
 330 #ifndef CLIB_MARCH_VARIANT
 331 int
 332 test_policer_add_del (u32 rx_sw_if_index, u8 * config_name, int is_add)
 333 {
 334   vnet_policer_main_t *pm = &vnet_policer_main;
 335   policer_read_response_type_st *template;
 336   policer_read_response_type_st *policer;
 337   vnet_hw_interface_t *rxhi;
 338   uword *p;
 339
 340   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 341
 342   /* Make sure caller didn't pass a vlan subif, etc. */
 343   if (rxhi->sw_if_index != rx_sw_if_index)
 344     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 345
 346   if (is_add)
 347     {
 348
 349       p = hash_get_mem (pm->policer_config_by_name, config_name);
 350
 351       if (p == 0)
 352         return -2;
 353
 354       template = pool_elt_at_index (pm->policer_templates, p[0]);
 355
 356       vnet_hw_interface_rx_redirect_to_node
 357         (pm->vnet_main, rxhi->hw_if_index, policer_by_sw_if_index_node.index);
 358
 359       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 360
 361       policer[0] = template[0];
 362
 363       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 364       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 365         = policer - pm->policers;
 366     }
 367   else
 368     {
 369       u32 pi;
 370       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 371                                              rxhi->hw_if_index,
 372                                              ~0 /* disable */ );
 373
 374       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 375       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 376       pool_put_index (pm->policers, pi);
 377     }
 378
 379   return 0;
 380 }
 381
 382 static clib_error_t *
 383 test_policer_command_fn (vlib_main_t * vm,
 384                          unformat_input_t * input, vlib_cli_command_t * cmd)
 385 {
 386   vnet_policer_main_t *pm = &vnet_policer_main;
 387   unformat_input_t _line_input, *line_input = &_line_input;
 388   u32 rx_sw_if_index;
 389   int rv;
 390   u8 *config_name = 0;
 391   int rx_set = 0;
 392   int is_add = 1;
 393   int is_show = 0;
 394   clib_error_t *error = NULL;
 395
 396   /* Get a line of input. */
 397   if (!unformat_user (input, unformat_line_input, line_input))
 398     return 0;
 399
 400   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 401     {
 402       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 403                     pm->vnet_main, &rx_sw_if_index))
 404         rx_set = 1;
 405       else if (unformat (line_input, "show"))
 406         is_show = 1;
 407       else if (unformat (line_input, "policer %s", &config_name))
 408         ;
 409       else if (unformat (line_input, "del"))
 410         is_add = 0;
 411       else
 412         break;
 413     }
 414
 415   if (rx_set == 0)
 416     {
 417       error = clib_error_return (0, "interface not set");
 418       goto done;
 419     }
 420
 421   if (is_show)
 422     {
 423       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 424       policer_read_response_type_st *policer;
 425       policer = pool_elt_at_index (pm->policers, pi);
 426
 427       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 428       goto done;
 429     }
 430
 431   if (is_add && config_name == 0)
 432     {
 433       error = clib_error_return (0, "policer config name required");
 434       goto done;
 435     }
 436
 437   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 438
 439   switch (rv)
 440     {
 441     case 0:
 442       break;
 443
 444     default:
 445       error = clib_error_return
 446         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 447       goto done;
 448     }
 449
 450 done:
 451   unformat_free (line_input);
 452
 453   return error;
 454 }
 455
 456 /* *INDENT-OFF* */
 457 VLIB_CLI_COMMAND (test_patch_command, static) = {
 458     .path = "test policer",
 459     .short_help =
 460     "intfc <intfc> policer <policer-config-name> [del]",
 461     .function = test_policer_command_fn,
 462 };
 463 /* *INDENT-ON* */
 464 #endif /* CLIB_MARCH_VARIANT */
 465
 466 #endif /* TEST_CODE */
 467
 468
 469 typedef struct
 470 {
 471   u32 sw_if_index;
 472   u32 next_index;
 473   u32 table_index;
 474   u32 offset;
 475   u32 policer_index;
 476 } policer_classify_trace_t;
 477
 478 static u8 *
 479 format_policer_classify_trace (u8 * s, va_list * args)
 480 {
 481   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 482   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 483   policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
 484
 485   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 486               " policer_index %d",
 487               t->sw_if_index, t->next_index, t->table_index, t->offset,
 488               t->policer_index);
 489   return s;
 490 }
 491
 492 #define foreach_policer_classify_error                 \
 493 _(MISS, "Policer classify misses")                     \
 494 _(HIT, "Policer classify hits")                        \
 495 _(CHAIN_HIT, "Policer classify hits after chain walk") \
 496 _(DROP, "Policer classify action drop")
 497
 498 typedef enum
 499 {
 500 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 501   foreach_policer_classify_error
 502 #undef _
 503     POLICER_CLASSIFY_N_ERROR,
 504 } policer_classify_error_t;
 505
 506 static char *policer_classify_error_strings[] = {
 507 #define _(sym,string) string,
 508   foreach_policer_classify_error
 509 #undef _
 510 };
 511
 512 static inline uword
 513 policer_classify_inline (vlib_main_t * vm,
 514                          vlib_node_runtime_t * node,
 515                          vlib_frame_t * frame,
 516                          policer_classify_table_id_t tid)
 517 {
 518   u32 n_left_from, *from, *to_next;
 519   policer_classify_next_index_t next_index;
 520   policer_classify_main_t *pcm = &policer_classify_main;
 521   vnet_classify_main_t *vcm = pcm->vnet_classify_main;
 522   f64 now = vlib_time_now (vm);
 523   u32 hits = 0;
 524   u32 misses = 0;
 525   u32 chain_hits = 0;
 526   u32 n_next_nodes;
 527   u64 time_in_policer_periods;
 528
 529   time_in_policer_periods =
 530     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 531
 532   n_next_nodes = node->n_next_nodes;
 533
 534   from = vlib_frame_vector_args (frame);
 535   n_left_from = frame->n_vectors;
 536
 537   /* First pass: compute hashes */
 538   while (n_left_from > 2)
 539     {
 540       vlib_buffer_t *b0, *b1;
 541       u32 bi0, bi1;
 542       u8 *h0, *h1;
 543       u32 sw_if_index0, sw_if_index1;
 544       u32 table_index0, table_index1;
 545       vnet_classify_table_t *t0, *t1;
 546
 547       /* Prefetch next iteration */
 548       {
 549         vlib_buffer_t *p1, *p2;
 550
 551         p1 = vlib_get_buffer (vm, from[1]);
 552         p2 = vlib_get_buffer (vm, from[2]);
 553
 554         vlib_prefetch_buffer_header (p1, STORE);
 555         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 556         vlib_prefetch_buffer_header (p2, STORE);
 557         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 558       }
 559
 560       bi0 = from[0];
 561       b0 = vlib_get_buffer (vm, bi0);
 562       h0 = b0->data;
 563
 564       bi1 = from[1];
 565       b1 = vlib_get_buffer (vm, bi1);
 566       h1 = b1->data;
 567
 568       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 569       table_index0 =
 570         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 571
 572       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 573       table_index1 =
 574         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 575
 576       t0 = pool_elt_at_index (vcm->tables, table_index0);
 577
 578       t1 = pool_elt_at_index (vcm->tables, table_index1);
 579
 580       vnet_buffer (b0)->l2_classify.hash =
 581         vnet_classify_hash_packet (t0, (u8 *) h0);
 582
 583       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 584
 585       vnet_buffer (b1)->l2_classify.hash =
 586         vnet_classify_hash_packet (t1, (u8 *) h1);
 587
 588       vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
 589
 590       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 591
 592       vnet_buffer (b1)->l2_classify.table_index = table_index1;
 593
 594       from += 2;
 595       n_left_from -= 2;
 596     }
 597
 598   while (n_left_from > 0)
 599     {
 600       vlib_buffer_t *b0;
 601       u32 bi0;
 602       u8 *h0;
 603       u32 sw_if_index0;
 604       u32 table_index0;
 605       vnet_classify_table_t *t0;
 606
 607       bi0 = from[0];
 608       b0 = vlib_get_buffer (vm, bi0);
 609       h0 = b0->data;
 610
 611       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 612       table_index0 =
 613         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 614
 615       t0 = pool_elt_at_index (vcm->tables, table_index0);
 616       vnet_buffer (b0)->l2_classify.hash =
 617         vnet_classify_hash_packet (t0, (u8 *) h0);
 618
 619       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 620       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 621
 622       from++;
 623       n_left_from--;
 624     }
 625
 626   next_index = node->cached_next_index;
 627   from = vlib_frame_vector_args (frame);
 628   n_left_from = frame->n_vectors;
 629
 630   while (n_left_from > 0)
 631     {
 632       u32 n_left_to_next;
 633
 634       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 635
 636       /* Not enough load/store slots to dual loop... */
 637       while (n_left_from > 0 && n_left_to_next > 0)
 638         {
 639           u32 bi0;
 640           vlib_buffer_t *b0;
 641           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 642           u32 table_index0;
 643           vnet_classify_table_t *t0;
 644           vnet_classify_entry_t *e0;
 645           u64 hash0;
 646           u8 *h0;
 647           u8 act0;
 648
 649           /* Stride 3 seems to work best */
 650           if (PREDICT_TRUE (n_left_from > 3))
 651             {
 652               vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
 653               vnet_classify_table_t *tp1;
 654               u32 table_index1;
 655               u64 phash1;
 656
 657               table_index1 = vnet_buffer (p1)->l2_classify.table_index;
 658
 659               if (PREDICT_TRUE (table_index1 != ~0))
 660                 {
 661                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 662                   phash1 = vnet_buffer (p1)->l2_classify.hash;
 663                   vnet_classify_prefetch_entry (tp1, phash1);
 664                 }
 665             }
 666
 667           /* Speculatively enqueue b0 to the current next frame */
 668           bi0 = from[0];
 669           to_next[0] = bi0;
 670           from += 1;
 671           to_next += 1;
 672           n_left_from -= 1;
 673           n_left_to_next -= 1;
 674
 675           b0 = vlib_get_buffer (vm, bi0);
 676           h0 = b0->data;
 677           table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 678           e0 = 0;
 679           t0 = 0;
 680
 681           if (tid == POLICER_CLASSIFY_TABLE_L2)
 682             {
 683               /* Feature bitmap update and determine the next node */
 684               next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
 685                                             L2INPUT_FEAT_POLICER_CLAS);
 686             }
 687           else
 688             vnet_get_config_data (pcm->vnet_config_main[tid],
 689                                   &b0->current_config_index, &next0,
 690                                   /* # bytes of config data */ 0);
 691
 692           vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 693
 694           if (PREDICT_TRUE (table_index0 != ~0))
 695             {
 696               hash0 = vnet_buffer (b0)->l2_classify.hash;
 697               t0 = pool_elt_at_index (vcm->tables, table_index0);
 698               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 699
 700               if (e0)
 701                 {
 702                   act0 = vnet_policer_police (vm,
 703                                               b0,
 704                                               e0->next_index,
 705                                               time_in_policer_periods,
 706                                               e0->opaque_index);
 707                   if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 708                     {
 709                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 710                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 711                     }
 712                   hits++;
 713                 }
 714               else
 715                 {
 716                   while (1)
 717                     {
 718                       if (PREDICT_TRUE (t0->next_table_index != ~0))
 719                         {
 720                           t0 = pool_elt_at_index (vcm->tables,
 721                                                   t0->next_table_index);
 722                         }
 723                       else
 724                         {
 725                           next0 = (t0->miss_next_index < n_next_nodes) ?
 726                             t0->miss_next_index : next0;
 727                           misses++;
 728                           break;
 729                         }
 730
 731                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 732                       e0 =
 733                         vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 734                       if (e0)
 735                         {
 736                           act0 = vnet_policer_police (vm,
 737                                                       b0,
 738                                                       e0->next_index,
 739                                                       time_in_policer_periods,
 740                                                       e0->opaque_index);
 741                           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 742                             {
 743                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 744                               b0->error =
 745                                 node->errors[POLICER_CLASSIFY_ERROR_DROP];
 746                             }
 747                           hits++;
 748                           chain_hits++;
 749                           break;
 750                         }
 751                     }
 752                 }
 753             }
 754           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 755                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 756             {
 757               policer_classify_trace_t *t =
 758                 vlib_add_trace (vm, node, b0, sizeof (*t));
 759               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 760               t->next_index = next0;
 761               t->table_index = t0 ? t0 - vcm->tables : ~0;
 762               t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
 763               t->policer_index = e0 ? e0->next_index : ~0;
 764             }
 765
 766           /* Verify speculative enqueue, maybe switch current next frame */
 767           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 768                                            n_left_to_next, bi0, next0);
 769         }
 770
 771       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 772     }
 773
 774   vlib_node_increment_counter (vm, node->node_index,
 775                                POLICER_CLASSIFY_ERROR_MISS, misses);
 776   vlib_node_increment_counter (vm, node->node_index,
 777                                POLICER_CLASSIFY_ERROR_HIT, hits);
 778   vlib_node_increment_counter (vm, node->node_index,
 779                                POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
 780
 781   return frame->n_vectors;
 782 }
 783
 784 VLIB_NODE_FN (ip4_policer_classify_node) (vlib_main_t * vm,
 785                                           vlib_node_runtime_t * node,
 786                                           vlib_frame_t * frame)
 787 {
 788   return policer_classify_inline (vm, node, frame,
 789                                   POLICER_CLASSIFY_TABLE_IP4);
 790 }
 791
 792 /* *INDENT-OFF* */
 793 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 794   .name = "ip4-policer-classify",
 795   .vector_size = sizeof (u32),
 796   .format_trace = format_policer_classify_trace,
 797   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 798   .error_strings = policer_classify_error_strings,
 799   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 800   .next_nodes = {
 801     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 802   },
 803 };
 804 /* *INDENT-ON* */
 805
 806 VLIB_NODE_FN (ip6_policer_classify_node) (vlib_main_t * vm,
 807                                           vlib_node_runtime_t * node,
 808                                           vlib_frame_t * frame)
 809 {
 810   return policer_classify_inline (vm, node, frame,
 811                                   POLICER_CLASSIFY_TABLE_IP6);
 812 }
 813
 814 /* *INDENT-OFF* */
 815 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 816   .name = "ip6-policer-classify",
 817   .vector_size = sizeof (u32),
 818   .format_trace = format_policer_classify_trace,
 819   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 820   .error_strings = policer_classify_error_strings,
 821   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 822   .next_nodes = {
 823     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 824   },
 825 };
 826 /* *INDENT-ON* */
 827
 828 VLIB_NODE_FN (l2_policer_classify_node) (vlib_main_t * vm,
 829                                          vlib_node_runtime_t * node,
 830                                          vlib_frame_t * frame)
 831 {
 832   return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 833 }
 834
 835 /* *INDENT-OFF* */
 836 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
 837   .name = "l2-policer-classify",
 838   .vector_size = sizeof (u32),
 839   .format_trace = format_policer_classify_trace,
 840   .n_errors = ARRAY_LEN (policer_classify_error_strings),
 841   .error_strings = policer_classify_error_strings,
 842   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 843   .next_nodes = {
 844     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 845   },
 846 };
 847 /* *INDENT-ON* */
 848
 849 #ifndef CLIB_MARCH_VARIANT
 850 static clib_error_t *
 851 policer_classify_init (vlib_main_t * vm)
 852 {
 853   policer_classify_main_t *pcm = &policer_classify_main;
 854
 855   pcm->vlib_main = vm;
 856   pcm->vnet_main = vnet_get_main ();
 857   pcm->vnet_classify_main = &vnet_classify_main;
 858
 859   /* Initialize L2 feature next-node indexes */
 860   feat_bitmap_init_next_nodes (vm,
 861                                l2_policer_classify_node.index,
 862                                L2INPUT_N_FEAT,
 863                                l2input_get_feat_names (),
 864                                pcm->feat_next_node_index);
 865
 866   return 0;
 867 }
 868
 869 VLIB_INIT_FUNCTION (policer_classify_init);
 870 #endif /* CLIB_MARCH_VARIANT */
 871
 872 /*
 873  * fd.io coding-style-patch-verification: ON
 874  *
 875  * Local Variables:
 876  * eval: (c-set-style "gnu")
 877  * End:
 878  */