src/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/policer/police_inlines.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/classify/policer_classify.h>
  24 #include <vnet/classify/vnet_classify.h>
  25 #include <vnet/l2/feat_bitmap.h>
  26 #include <vnet/l2/l2_input.h>
  27
  28
  29 /* Dispatch functions meant to be instantiated elsewhere */
  30
  31 typedef struct
  32 {
  33   u32 next_index;
  34   u32 sw_if_index;
  35   u32 policer_index;
  36 } vnet_policer_trace_t;
  37
  38 /* packet trace format function */
  39 static u8 *
  40 format_policer_trace (u8 * s, va_list * args)
  41 {
  42   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  43   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  44   vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
  45
  46   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  47               t->sw_if_index, t->policer_index, t->next_index);
  48   return s;
  49 }
  50
  51 #define foreach_vnet_policer_error              \
  52 _(TRANSMIT, "Packets Transmitted")              \
  53 _(DROP, "Packets Dropped")
  54
  55 typedef enum
  56 {
  57 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  58   foreach_vnet_policer_error
  59 #undef _
  60     VNET_POLICER_N_ERROR,
  61 } vnet_policer_error_t;
  62
  63 static char *vnet_policer_error_strings[] = {
  64 #define _(sym,string) string,
  65   foreach_vnet_policer_error
  66 #undef _
  67 };
  68
  69 static inline uword
  70 vnet_policer_inline (vlib_main_t * vm,
  71                      vlib_node_runtime_t * node,
  72                      vlib_frame_t * frame, vnet_policer_index_t which)
  73 {
  74   u32 n_left_from, *from, *to_next;
  75   vnet_policer_next_t next_index;
  76   vnet_policer_main_t *pm = &vnet_policer_main;
  77   u64 time_in_policer_periods;
  78   u32 transmitted = 0;
  79
  80   time_in_policer_periods =
  81     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
  82
  83   from = vlib_frame_vector_args (frame);
  84   n_left_from = frame->n_vectors;
  85   next_index = node->cached_next_index;
  86
  87   while (n_left_from > 0)
  88     {
  89       u32 n_left_to_next;
  90
  91       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  92
  93       while (n_left_from >= 4 && n_left_to_next >= 2)
  94         {
  95           u32 bi0, bi1;
  96           vlib_buffer_t *b0, *b1;
  97           u32 next0, next1;
  98           u32 sw_if_index0, sw_if_index1;
  99           u32 pi0 = 0, pi1 = 0;
 100           u8 act0, act1;
 101
 102           /* Prefetch next iteration. */
 103           {
 104             vlib_buffer_t *b2, *b3;
 105
 106             b2 = vlib_get_buffer (vm, from[2]);
 107             b3 = vlib_get_buffer (vm, from[3]);
 108
 109             vlib_prefetch_buffer_header (b2, LOAD);
 110             vlib_prefetch_buffer_header (b3, LOAD);
 111           }
 112
 113           /* speculatively enqueue b0 and b1 to the current next frame */
 114           to_next[0] = bi0 = from[0];
 115           to_next[1] = bi1 = from[1];
 116           from += 2;
 117           to_next += 2;
 118           n_left_from -= 2;
 119           n_left_to_next -= 2;
 120
 121           b0 = vlib_get_buffer (vm, bi0);
 122           b1 = vlib_get_buffer (vm, bi1);
 123
 124           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 125           next0 = VNET_POLICER_NEXT_TRANSMIT;
 126
 127           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 128           next1 = VNET_POLICER_NEXT_TRANSMIT;
 129
 130
 131           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 132             {
 133               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 134               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 135             }
 136
 137           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 138             {
 139               pi0 = vnet_buffer (b0)->policer.index;
 140               pi1 = vnet_buffer (b1)->policer.index;
 141             }
 142
 143           if (which == VNET_POLICER_INDEX_BY_EITHER)
 144             {
 145               pi0 = vnet_buffer (b0)->policer.index;
 146               pi0 = (pi0 != ~0) ? pi0 :
 147                 pm->policer_index_by_sw_if_index[sw_if_index0];
 148               pi1 = vnet_buffer (b1)->policer.index;
 149               pi1 = (pi1 != ~0) ? pi1 :
 150                 pm->policer_index_by_sw_if_index[sw_if_index1];
 151             }
 152
 153           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 154                                       POLICE_CONFORM /* no chaining */ );
 155
 156           act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
 157                                       POLICE_CONFORM /* no chaining */ );
 158
 159           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 160             {
 161               next0 = VNET_POLICER_NEXT_DROP;
 162               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 163             }
 164           else                  /* transmit or mark-and-transmit action */
 165             {
 166               transmitted++;
 167             }
 168
 169           if (PREDICT_FALSE (act1 == SSE2_QOS_ACTION_DROP))     /* drop action */
 170             {
 171               next1 = VNET_POLICER_NEXT_DROP;
 172               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 173             }
 174           else                  /* transmit or mark-and-transmit action */
 175             {
 176               transmitted++;
 177             }
 178
 179
 180           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
 181             {
 182               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 183                 {
 184                   vnet_policer_trace_t *t =
 185                     vlib_add_trace (vm, node, b0, sizeof (*t));
 186                   t->sw_if_index = sw_if_index0;
 187                   t->next_index = next0;
 188                 }
 189               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 190                 {
 191                   vnet_policer_trace_t *t =
 192                     vlib_add_trace (vm, node, b1, sizeof (*t));
 193                   t->sw_if_index = sw_if_index1;
 194                   t->next_index = next1;
 195                 }
 196             }
 197
 198           /* verify speculative enqueues, maybe switch current next frame */
 199           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 200                                            to_next, n_left_to_next,
 201                                            bi0, bi1, next0, next1);
 202         }
 203
 204       while (n_left_from > 0 && n_left_to_next > 0)
 205         {
 206           u32 bi0;
 207           vlib_buffer_t *b0;
 208           u32 next0;
 209           u32 sw_if_index0;
 210           u32 pi0 = 0;
 211           u8 act0;
 212
 213           bi0 = from[0];
 214           to_next[0] = bi0;
 215           from += 1;
 216           to_next += 1;
 217           n_left_from -= 1;
 218           n_left_to_next -= 1;
 219
 220           b0 = vlib_get_buffer (vm, bi0);
 221
 222           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 223           next0 = VNET_POLICER_NEXT_TRANSMIT;
 224
 225           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 226             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 227
 228           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 229             pi0 = vnet_buffer (b0)->policer.index;
 230
 231           if (which == VNET_POLICER_INDEX_BY_EITHER)
 232             {
 233               pi0 = vnet_buffer (b0)->policer.index;
 234               pi0 = (pi0 != ~0) ? pi0 :
 235                 pm->policer_index_by_sw_if_index[sw_if_index0];
 236             }
 237
 238           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 239                                       POLICE_CONFORM /* no chaining */ );
 240
 241           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 242             {
 243               next0 = VNET_POLICER_NEXT_DROP;
 244               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 245             }
 246           else                  /* transmit or mark-and-transmit action */
 247             {
 248               transmitted++;
 249             }
 250
 251           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 252                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 253             {
 254               vnet_policer_trace_t *t =
 255                 vlib_add_trace (vm, node, b0, sizeof (*t));
 256               t->sw_if_index = sw_if_index0;
 257               t->next_index = next0;
 258               t->policer_index = pi0;
 259             }
 260
 261           /* verify speculative enqueue, maybe switch current next frame */
 262           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 263                                            to_next, n_left_to_next,
 264                                            bi0, next0);
 265         }
 266
 267       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 268     }
 269
 270   vlib_node_increment_counter (vm, node->node_index,
 271                                VNET_POLICER_ERROR_TRANSMIT, transmitted);
 272   return frame->n_vectors;
 273 }
 274
 275 uword
 276 vnet_policer_by_sw_if_index (vlib_main_t * vm,
 277                              vlib_node_runtime_t * node, vlib_frame_t * frame)
 278 {
 279   return vnet_policer_inline (vm, node, frame,
 280                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 281 }
 282
 283 uword
 284 vnet_policer_by_opaque (vlib_main_t * vm,
 285                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 286 {
 287   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_OPAQUE);
 288 }
 289
 290 uword
 291 vnet_policer_by_either (vlib_main_t * vm,
 292                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 293 {
 294   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_EITHER);
 295 }
 296
 297 void
 298 vnet_policer_node_funcs_reference (void)
 299 {
 300 }
 301
 302
 303 #define TEST_CODE 1
 304
 305 #ifdef TEST_CODE
 306
 307 /* *INDENT-OFF* */
 308 VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = {
 309   .function = vnet_policer_by_sw_if_index,
 310   .name = "policer-by-sw-if-index",
 311   .vector_size = sizeof (u32),
 312   .format_trace = format_policer_trace,
 313   .type = VLIB_NODE_TYPE_INTERNAL,
 314
 315   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 316   .error_strings = vnet_policer_error_strings,
 317
 318   .n_next_nodes = VNET_POLICER_N_NEXT,
 319
 320   /* edit / add dispositions here */
 321   .next_nodes = {
 322     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 323     [VNET_POLICER_NEXT_DROP] = "error-drop",
 324   },
 325 };
 326
 327 VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
 328                               vnet_policer_by_sw_if_index);
 329 /* *INDENT-ON* */
 330
 331
 332 int
 333 test_policer_add_del (u32 rx_sw_if_index, u8 * config_name, int is_add)
 334 {
 335   vnet_policer_main_t *pm = &vnet_policer_main;
 336   policer_read_response_type_st *template;
 337   policer_read_response_type_st *policer;
 338   vnet_hw_interface_t *rxhi;
 339   uword *p;
 340
 341   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 342
 343   /* Make sure caller didn't pass a vlan subif, etc. */
 344   if (rxhi->sw_if_index != rx_sw_if_index)
 345     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 346
 347   if (is_add)
 348     {
 349
 350       p = hash_get_mem (pm->policer_config_by_name, config_name);
 351
 352       if (p == 0)
 353         return -2;
 354
 355       template = pool_elt_at_index (pm->policer_templates, p[0]);
 356
 357       vnet_hw_interface_rx_redirect_to_node
 358         (pm->vnet_main, rxhi->hw_if_index, policer_by_sw_if_index_node.index);
 359
 360       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 361
 362       policer[0] = template[0];
 363
 364       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 365       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 366         = policer - pm->policers;
 367     }
 368   else
 369     {
 370       u32 pi;
 371       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 372                                              rxhi->hw_if_index,
 373                                              ~0 /* disable */ );
 374
 375       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 376       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 377       pool_put_index (pm->policers, pi);
 378     }
 379
 380   return 0;
 381 }
 382
 383 static clib_error_t *
 384 test_policer_command_fn (vlib_main_t * vm,
 385                          unformat_input_t * input, vlib_cli_command_t * cmd)
 386 {
 387   vnet_policer_main_t *pm = &vnet_policer_main;
 388   unformat_input_t _line_input, *line_input = &_line_input;
 389   u32 rx_sw_if_index;
 390   int rv;
 391   u8 *config_name = 0;
 392   int rx_set = 0;
 393   int is_add = 1;
 394   int is_show = 0;
 395   clib_error_t *error = NULL;
 396
 397   /* Get a line of input. */
 398   if (!unformat_user (input, unformat_line_input, line_input))
 399     return 0;
 400
 401   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 402     {
 403       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 404                     pm->vnet_main, &rx_sw_if_index))
 405         rx_set = 1;
 406       else if (unformat (line_input, "show"))
 407         is_show = 1;
 408       else if (unformat (line_input, "policer %s", &config_name))
 409         ;
 410       else if (unformat (line_input, "del"))
 411         is_add = 0;
 412       else
 413         break;
 414     }
 415
 416   if (rx_set == 0)
 417     {
 418       error = clib_error_return (0, "interface not set");
 419       goto done;
 420     }
 421
 422   if (is_show)
 423     {
 424       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 425       policer_read_response_type_st *policer;
 426       policer = pool_elt_at_index (pm->policers, pi);
 427
 428       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 429       goto done;
 430     }
 431
 432   if (is_add && config_name == 0)
 433     {
 434       error = clib_error_return (0, "policer config name required");
 435       goto done;
 436     }
 437
 438   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 439
 440   switch (rv)
 441     {
 442     case 0:
 443       break;
 444
 445     default:
 446       error = clib_error_return
 447         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 448       goto done;
 449     }
 450
 451 done:
 452   unformat_free (line_input);
 453
 454   return error;
 455 }
 456
 457 /* *INDENT-OFF* */
 458 VLIB_CLI_COMMAND (test_patch_command, static) = {
 459     .path = "test policer",
 460     .short_help =
 461     "intfc <intfc> policer <policer-config-name> [del]",
 462     .function = test_policer_command_fn,
 463 };
 464 /* *INDENT-ON* */
 465
 466 #endif /* TEST_CODE */
 467
 468
 469 typedef struct
 470 {
 471   u32 sw_if_index;
 472   u32 next_index;
 473   u32 table_index;
 474   u32 offset;
 475   u32 policer_index;
 476 } policer_classify_trace_t;
 477
 478 static u8 *
 479 format_policer_classify_trace (u8 * s, va_list * args)
 480 {
 481   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 482   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 483   policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
 484
 485   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 486               " policer_index %d",
 487               t->sw_if_index, t->next_index, t->table_index, t->offset,
 488               t->policer_index);
 489   return s;
 490 }
 491
 492 #define foreach_policer_classify_error                 \
 493 _(MISS, "Policer classify misses")                     \
 494 _(HIT, "Policer classify hits")                        \
 495 _(CHAIN_HIT, "Policer classify hits after chain walk") \
 496 _(DROP, "Policer classify action drop")
 497
 498 typedef enum
 499 {
 500 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 501   foreach_policer_classify_error
 502 #undef _
 503     POLICER_CLASSIFY_N_ERROR,
 504 } policer_classify_error_t;
 505
 506 static char *policer_classify_error_strings[] = {
 507 #define _(sym,string) string,
 508   foreach_policer_classify_error
 509 #undef _
 510 };
 511
 512 static inline uword
 513 policer_classify_inline (vlib_main_t * vm,
 514                          vlib_node_runtime_t * node,
 515                          vlib_frame_t * frame,
 516                          policer_classify_table_id_t tid)
 517 {
 518   u32 n_left_from, *from, *to_next;
 519   policer_classify_next_index_t next_index;
 520   policer_classify_main_t *pcm = &policer_classify_main;
 521   vnet_classify_main_t *vcm = pcm->vnet_classify_main;
 522   f64 now = vlib_time_now (vm);
 523   u32 hits = 0;
 524   u32 misses = 0;
 525   u32 chain_hits = 0;
 526   u32 drop = 0;
 527   u32 n_next_nodes;
 528   u64 time_in_policer_periods;
 529
 530   time_in_policer_periods =
 531     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 532
 533   n_next_nodes = node->n_next_nodes;
 534
 535   from = vlib_frame_vector_args (frame);
 536   n_left_from = frame->n_vectors;
 537
 538   /* First pass: compute hashes */
 539   while (n_left_from > 2)
 540     {
 541       vlib_buffer_t *b0, *b1;
 542       u32 bi0, bi1;
 543       u8 *h0, *h1;
 544       u32 sw_if_index0, sw_if_index1;
 545       u32 table_index0, table_index1;
 546       vnet_classify_table_t *t0, *t1;
 547
 548       /* Prefetch next iteration */
 549       {
 550         vlib_buffer_t *p1, *p2;
 551
 552         p1 = vlib_get_buffer (vm, from[1]);
 553         p2 = vlib_get_buffer (vm, from[2]);
 554
 555         vlib_prefetch_buffer_header (p1, STORE);
 556         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 557         vlib_prefetch_buffer_header (p2, STORE);
 558         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 559       }
 560
 561       bi0 = from[0];
 562       b0 = vlib_get_buffer (vm, bi0);
 563       h0 = b0->data;
 564
 565       bi1 = from[1];
 566       b1 = vlib_get_buffer (vm, bi1);
 567       h1 = b1->data;
 568
 569       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 570       table_index0 =
 571         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 572
 573       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 574       table_index1 =
 575         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 576
 577       t0 = pool_elt_at_index (vcm->tables, table_index0);
 578
 579       t1 = pool_elt_at_index (vcm->tables, table_index1);
 580
 581       vnet_buffer (b0)->l2_classify.hash =
 582         vnet_classify_hash_packet (t0, (u8 *) h0);
 583
 584       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 585
 586       vnet_buffer (b1)->l2_classify.hash =
 587         vnet_classify_hash_packet (t1, (u8 *) h1);
 588
 589       vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
 590
 591       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 592
 593       vnet_buffer (b1)->l2_classify.table_index = table_index1;
 594
 595       from += 2;
 596       n_left_from -= 2;
 597     }
 598
 599   while (n_left_from > 0)
 600     {
 601       vlib_buffer_t *b0;
 602       u32 bi0;
 603       u8 *h0;
 604       u32 sw_if_index0;
 605       u32 table_index0;
 606       vnet_classify_table_t *t0;
 607
 608       bi0 = from[0];
 609       b0 = vlib_get_buffer (vm, bi0);
 610       h0 = b0->data;
 611
 612       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 613       table_index0 =
 614         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 615
 616       t0 = pool_elt_at_index (vcm->tables, table_index0);
 617       vnet_buffer (b0)->l2_classify.hash =
 618         vnet_classify_hash_packet (t0, (u8 *) h0);
 619
 620       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 621       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 622
 623       from++;
 624       n_left_from--;
 625     }
 626
 627   next_index = node->cached_next_index;
 628   from = vlib_frame_vector_args (frame);
 629   n_left_from = frame->n_vectors;
 630
 631   while (n_left_from > 0)
 632     {
 633       u32 n_left_to_next;
 634
 635       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 636
 637       /* Not enough load/store slots to dual loop... */
 638       while (n_left_from > 0 && n_left_to_next > 0)
 639         {
 640           u32 bi0;
 641           vlib_buffer_t *b0;
 642           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 643           u32 table_index0;
 644           vnet_classify_table_t *t0;
 645           vnet_classify_entry_t *e0;
 646           u64 hash0;
 647           u8 *h0;
 648           u8 act0;
 649
 650           /* Stride 3 seems to work best */
 651           if (PREDICT_TRUE (n_left_from > 3))
 652             {
 653               vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
 654               vnet_classify_table_t *tp1;
 655               u32 table_index1;
 656               u64 phash1;
 657
 658               table_index1 = vnet_buffer (p1)->l2_classify.table_index;
 659
 660               if (PREDICT_TRUE (table_index1 != ~0))
 661                 {
 662                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 663                   phash1 = vnet_buffer (p1)->l2_classify.hash;
 664                   vnet_classify_prefetch_entry (tp1, phash1);
 665                 }
 666             }
 667
 668           /* Speculatively enqueue b0 to the current next frame */
 669           bi0 = from[0];
 670           to_next[0] = bi0;
 671           from += 1;
 672           to_next += 1;
 673           n_left_from -= 1;
 674           n_left_to_next -= 1;
 675
 676           b0 = vlib_get_buffer (vm, bi0);
 677           h0 = b0->data;
 678           table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 679           e0 = 0;
 680           t0 = 0;
 681
 682           if (tid == POLICER_CLASSIFY_TABLE_L2)
 683             {
 684               /* Feature bitmap update and determine the next node */
 685               next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
 686                                             L2INPUT_FEAT_POLICER_CLAS);
 687             }
 688           else
 689             vnet_get_config_data (pcm->vnet_config_main[tid],
 690                                   &b0->current_config_index, &next0,
 691                                   /* # bytes of config data */ 0);
 692
 693           vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 694
 695           if (PREDICT_TRUE (table_index0 != ~0))
 696             {
 697               hash0 = vnet_buffer (b0)->l2_classify.hash;
 698               t0 = pool_elt_at_index (vcm->tables, table_index0);
 699               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 700
 701               if (e0)
 702                 {
 703                   act0 = vnet_policer_police (vm,
 704                                               b0,
 705                                               e0->next_index,
 706                                               time_in_policer_periods,
 707                                               e0->opaque_index);
 708                   if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 709                     {
 710                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 711                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 712                       drop++;
 713                     }
 714                   hits++;
 715                 }
 716               else
 717                 {
 718                   while (1)
 719                     {
 720                       if (PREDICT_TRUE (t0->next_table_index != ~0))
 721                         {
 722                           t0 = pool_elt_at_index (vcm->tables,
 723                                                   t0->next_table_index);
 724                         }
 725                       else
 726                         {
 727                           next0 = (t0->miss_next_index < n_next_nodes) ?
 728                             t0->miss_next_index : next0;
 729                           misses++;
 730                           break;
 731                         }
 732
 733                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 734                       e0 =
 735                         vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 736                       if (e0)
 737                         {
 738                           act0 = vnet_policer_police (vm,
 739                                                       b0,
 740                                                       e0->next_index,
 741                                                       time_in_policer_periods,
 742                                                       e0->opaque_index);
 743                           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 744                             {
 745                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 746                               b0->error =
 747                                 node->errors[POLICER_CLASSIFY_ERROR_DROP];
 748                               drop++;
 749                             }
 750                           hits++;
 751                           chain_hits++;
 752                           break;
 753                         }
 754                     }
 755                 }
 756             }
 757           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 758                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 759             {
 760               policer_classify_trace_t *t =
 761                 vlib_add_trace (vm, node, b0, sizeof (*t));
 762               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 763               t->next_index = next0;
 764               t->table_index = t0 ? t0 - vcm->tables : ~0;
 765               t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
 766               t->policer_index = e0 ? e0->next_index : ~0;
 767             }
 768
 769           /* Verify speculative enqueue, maybe switch current next frame */
 770           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 771                                            n_left_to_next, bi0, next0);
 772         }
 773
 774       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 775     }
 776
 777   vlib_node_increment_counter (vm, node->node_index,
 778                                POLICER_CLASSIFY_ERROR_MISS, misses);
 779   vlib_node_increment_counter (vm, node->node_index,
 780                                POLICER_CLASSIFY_ERROR_HIT, hits);
 781   vlib_node_increment_counter (vm, node->node_index,
 782                                POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
 783   vlib_node_increment_counter (vm, node->node_index,
 784                                POLICER_CLASSIFY_ERROR_DROP, drop);
 785
 786   return frame->n_vectors;
 787 }
 788
 789 static uword
 790 ip4_policer_classify (vlib_main_t * vm,
 791                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 792 {
 793   return policer_classify_inline (vm, node, frame,
 794                                   POLICER_CLASSIFY_TABLE_IP4);
 795 }
 796
 797 /* *INDENT-OFF* */
 798 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 799   .function = ip4_policer_classify,
 800   .name = "ip4-policer-classify",
 801   .vector_size = sizeof (u32),
 802   .format_trace = format_policer_classify_trace,
 803   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 804   .error_strings = policer_classify_error_strings,
 805   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 806   .next_nodes = {
 807     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 808   },
 809 };
 810
 811 VLIB_NODE_FUNCTION_MULTIARCH (ip4_policer_classify_node, ip4_policer_classify);
 812 /* *INDENT-ON* */
 813
 814 static uword
 815 ip6_policer_classify (vlib_main_t * vm,
 816                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 817 {
 818   return policer_classify_inline (vm, node, frame,
 819                                   POLICER_CLASSIFY_TABLE_IP6);
 820 }
 821
 822 /* *INDENT-OFF* */
 823 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 824   .function = ip6_policer_classify,
 825   .name = "ip6-policer-classify",
 826   .vector_size = sizeof (u32),
 827   .format_trace = format_policer_classify_trace,
 828   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 829   .error_strings = policer_classify_error_strings,
 830   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 831   .next_nodes = {
 832     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 833   },
 834 };
 835
 836 VLIB_NODE_FUNCTION_MULTIARCH (ip6_policer_classify_node, ip6_policer_classify);
 837 /* *INDENT-ON* */
 838
 839 static uword
 840 l2_policer_classify (vlib_main_t * vm,
 841                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 842 {
 843   return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 844 }
 845
 846 /* *INDENT-OFF* */
 847 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
 848   .function = l2_policer_classify,
 849   .name = "l2-policer-classify",
 850   .vector_size = sizeof (u32),
 851   .format_trace = format_policer_classify_trace,
 852   .n_errors = ARRAY_LEN (policer_classify_error_strings),
 853   .error_strings = policer_classify_error_strings,
 854   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 855   .next_nodes = {
 856     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 857   },
 858 };
 859
 860 VLIB_NODE_FUNCTION_MULTIARCH (l2_policer_classify_node, l2_policer_classify);
 861 /* *INDENT-ON* */
 862
 863 static clib_error_t *
 864 policer_classify_init (vlib_main_t * vm)
 865 {
 866   policer_classify_main_t *pcm = &policer_classify_main;
 867
 868   pcm->vlib_main = vm;
 869   pcm->vnet_main = vnet_get_main ();
 870   pcm->vnet_classify_main = &vnet_classify_main;
 871
 872   /* Initialize L2 feature next-node indexes */
 873   feat_bitmap_init_next_nodes (vm,
 874                                l2_policer_classify_node.index,
 875                                L2INPUT_N_FEAT,
 876                                l2input_get_feat_names (),
 877                                pcm->feat_next_node_index);
 878
 879   return 0;
 880 }
 881
 882 VLIB_INIT_FUNCTION (policer_classify_init);
 883
 884 /*
 885  * fd.io coding-style-patch-verification: ON
 886  *
 887  * Local Variables:
 888  * eval: (c-set-style "gnu")
 889  * End:
 890  */