src/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/policer/police_inlines.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/classify/policer_classify.h>
  24 #include <vnet/classify/vnet_classify.h>
  25 #include <vnet/l2/feat_bitmap.h>
  26 #include <vnet/l2/l2_input.h>
  27
  28
  29 /* Dispatch functions meant to be instantiated elsewhere */
  30
  31 typedef struct
  32 {
  33   u32 next_index;
  34   u32 sw_if_index;
  35   u32 policer_index;
  36 } vnet_policer_trace_t;
  37
  38 /* packet trace format function */
  39 static u8 *
  40 format_policer_trace (u8 * s, va_list * args)
  41 {
  42   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  43   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  44   vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
  45
  46   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  47               t->sw_if_index, t->policer_index, t->next_index);
  48   return s;
  49 }
  50
  51 #define foreach_vnet_policer_error              \
  52 _(TRANSMIT, "Packets Transmitted")              \
  53 _(DROP, "Packets Dropped")
  54
  55 typedef enum
  56 {
  57 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  58   foreach_vnet_policer_error
  59 #undef _
  60     VNET_POLICER_N_ERROR,
  61 } vnet_policer_error_t;
  62
  63 static char *vnet_policer_error_strings[] = {
  64 #define _(sym,string) string,
  65   foreach_vnet_policer_error
  66 #undef _
  67 };
  68
  69 static inline uword
  70 vnet_policer_inline (vlib_main_t * vm,
  71                      vlib_node_runtime_t * node,
  72                      vlib_frame_t * frame, vnet_policer_index_t which)
  73 {
  74   u32 n_left_from, *from, *to_next;
  75   vnet_policer_next_t next_index;
  76   vnet_policer_main_t *pm = &vnet_policer_main;
  77   u64 time_in_policer_periods;
  78   u32 transmitted = 0;
  79
  80   time_in_policer_periods =
  81     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
  82
  83   from = vlib_frame_vector_args (frame);
  84   n_left_from = frame->n_vectors;
  85   next_index = node->cached_next_index;
  86
  87   while (n_left_from > 0)
  88     {
  89       u32 n_left_to_next;
  90
  91       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  92
  93       while (n_left_from >= 4 && n_left_to_next >= 2)
  94         {
  95           u32 bi0, bi1;
  96           vlib_buffer_t *b0, *b1;
  97           u32 next0, next1;
  98           u32 sw_if_index0, sw_if_index1;
  99           u32 pi0 = 0, pi1 = 0;
 100           u8 act0, act1;
 101
 102           /* Prefetch next iteration. */
 103           {
 104             vlib_buffer_t *b2, *b3;
 105
 106             b2 = vlib_get_buffer (vm, from[2]);
 107             b3 = vlib_get_buffer (vm, from[3]);
 108
 109             vlib_prefetch_buffer_header (b2, LOAD);
 110             vlib_prefetch_buffer_header (b3, LOAD);
 111           }
 112
 113           /* speculatively enqueue b0 and b1 to the current next frame */
 114           to_next[0] = bi0 = from[0];
 115           to_next[1] = bi1 = from[1];
 116           from += 2;
 117           to_next += 2;
 118           n_left_from -= 2;
 119           n_left_to_next -= 2;
 120
 121           b0 = vlib_get_buffer (vm, bi0);
 122           b1 = vlib_get_buffer (vm, bi1);
 123
 124           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 125           next0 = VNET_POLICER_NEXT_TRANSMIT;
 126
 127           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 128           next1 = VNET_POLICER_NEXT_TRANSMIT;
 129
 130
 131           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 132             {
 133               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 134               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 135             }
 136
 137           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 138             {
 139               pi0 = vnet_buffer (b0)->policer.index;
 140               pi1 = vnet_buffer (b1)->policer.index;
 141             }
 142
 143           if (which == VNET_POLICER_INDEX_BY_EITHER)
 144             {
 145               pi0 = vnet_buffer (b0)->policer.index;
 146               pi0 = (pi0 != ~0) ? pi0 :
 147                 pm->policer_index_by_sw_if_index[sw_if_index0];
 148               pi1 = vnet_buffer (b1)->policer.index;
 149               pi1 = (pi1 != ~0) ? pi1 :
 150                 pm->policer_index_by_sw_if_index[sw_if_index1];
 151             }
 152
 153           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 154                                       POLICE_CONFORM /* no chaining */ );
 155
 156           act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
 157                                       POLICE_CONFORM /* no chaining */ );
 158
 159           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 160             {
 161               next0 = VNET_POLICER_NEXT_DROP;
 162               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 163             }
 164           else                  /* transmit or mark-and-transmit action */
 165             {
 166               transmitted++;
 167             }
 168
 169           if (PREDICT_FALSE (act1 == SSE2_QOS_ACTION_DROP))     /* drop action */
 170             {
 171               next1 = VNET_POLICER_NEXT_DROP;
 172               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 173             }
 174           else                  /* transmit or mark-and-transmit action */
 175             {
 176               transmitted++;
 177             }
 178
 179
 180           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
 181             {
 182               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 183                 {
 184                   vnet_policer_trace_t *t =
 185                     vlib_add_trace (vm, node, b0, sizeof (*t));
 186                   t->sw_if_index = sw_if_index0;
 187                   t->next_index = next0;
 188                 }
 189               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 190                 {
 191                   vnet_policer_trace_t *t =
 192                     vlib_add_trace (vm, node, b1, sizeof (*t));
 193                   t->sw_if_index = sw_if_index1;
 194                   t->next_index = next1;
 195                 }
 196             }
 197
 198           /* verify speculative enqueues, maybe switch current next frame */
 199           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 200                                            to_next, n_left_to_next,
 201                                            bi0, bi1, next0, next1);
 202         }
 203
 204       while (n_left_from > 0 && n_left_to_next > 0)
 205         {
 206           u32 bi0;
 207           vlib_buffer_t *b0;
 208           u32 next0;
 209           u32 sw_if_index0;
 210           u32 pi0 = 0;
 211           u8 act0;
 212
 213           bi0 = from[0];
 214           to_next[0] = bi0;
 215           from += 1;
 216           to_next += 1;
 217           n_left_from -= 1;
 218           n_left_to_next -= 1;
 219
 220           b0 = vlib_get_buffer (vm, bi0);
 221
 222           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 223           next0 = VNET_POLICER_NEXT_TRANSMIT;
 224
 225           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 226             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 227
 228           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 229             pi0 = vnet_buffer (b0)->policer.index;
 230
 231           if (which == VNET_POLICER_INDEX_BY_EITHER)
 232             {
 233               pi0 = vnet_buffer (b0)->policer.index;
 234               pi0 = (pi0 != ~0) ? pi0 :
 235                 pm->policer_index_by_sw_if_index[sw_if_index0];
 236             }
 237
 238           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 239                                       POLICE_CONFORM /* no chaining */ );
 240
 241           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 242             {
 243               next0 = VNET_POLICER_NEXT_DROP;
 244               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 245             }
 246           else                  /* transmit or mark-and-transmit action */
 247             {
 248               transmitted++;
 249             }
 250
 251           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 252                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 253             {
 254               vnet_policer_trace_t *t =
 255                 vlib_add_trace (vm, node, b0, sizeof (*t));
 256               t->sw_if_index = sw_if_index0;
 257               t->next_index = next0;
 258               t->policer_index = pi0;
 259             }
 260
 261           /* verify speculative enqueue, maybe switch current next frame */
 262           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 263                                            to_next, n_left_to_next,
 264                                            bi0, next0);
 265         }
 266
 267       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 268     }
 269
 270   vlib_node_increment_counter (vm, node->node_index,
 271                                VNET_POLICER_ERROR_TRANSMIT, transmitted);
 272   return frame->n_vectors;
 273 }
 274
 275 uword
 276 vnet_policer_by_sw_if_index (vlib_main_t * vm,
 277                              vlib_node_runtime_t * node, vlib_frame_t * frame)
 278 {
 279   return vnet_policer_inline (vm, node, frame,
 280                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 281 }
 282
 283 uword
 284 vnet_policer_by_opaque (vlib_main_t * vm,
 285                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 286 {
 287   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_OPAQUE);
 288 }
 289
 290 uword
 291 vnet_policer_by_either (vlib_main_t * vm,
 292                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 293 {
 294   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_EITHER);
 295 }
 296
 297 void
 298 vnet_policer_node_funcs_reference (void)
 299 {
 300 }
 301
 302
 303 #define TEST_CODE 1
 304
 305 #ifdef TEST_CODE
 306
 307 /* *INDENT-OFF* */
 308 VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = {
 309   .function = vnet_policer_by_sw_if_index,
 310   .name = "policer-by-sw-if-index",
 311   .vector_size = sizeof (u32),
 312   .format_trace = format_policer_trace,
 313   .type = VLIB_NODE_TYPE_INTERNAL,
 314
 315   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 316   .error_strings = vnet_policer_error_strings,
 317
 318   .n_next_nodes = VNET_POLICER_N_NEXT,
 319
 320   /* edit / add dispositions here */
 321   .next_nodes = {
 322     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 323     [VNET_POLICER_NEXT_DROP] = "error-drop",
 324   },
 325 };
 326
 327 VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
 328                               vnet_policer_by_sw_if_index);
 329 /* *INDENT-ON* */
 330
 331
 332 int
 333 test_policer_add_del (u32 rx_sw_if_index, u8 * config_name, int is_add)
 334 {
 335   vnet_policer_main_t *pm = &vnet_policer_main;
 336   policer_read_response_type_st *template;
 337   policer_read_response_type_st *policer;
 338   vnet_hw_interface_t *rxhi;
 339   uword *p;
 340
 341   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 342
 343   /* Make sure caller didn't pass a vlan subif, etc. */
 344   if (rxhi->sw_if_index != rx_sw_if_index)
 345     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 346
 347   if (is_add)
 348     {
 349
 350       p = hash_get_mem (pm->policer_config_by_name, config_name);
 351
 352       if (p == 0)
 353         return -2;
 354
 355       template = pool_elt_at_index (pm->policer_templates, p[0]);
 356
 357       vnet_hw_interface_rx_redirect_to_node
 358         (pm->vnet_main, rxhi->hw_if_index, policer_by_sw_if_index_node.index);
 359
 360       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 361
 362       policer[0] = template[0];
 363
 364       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 365       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 366         = policer - pm->policers;
 367     }
 368   else
 369     {
 370       u32 pi;
 371       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 372                                              rxhi->hw_if_index,
 373                                              ~0 /* disable */ );
 374
 375       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 376       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 377       pool_put_index (pm->policers, pi);
 378     }
 379
 380   return 0;
 381 }
 382
 383 static clib_error_t *
 384 test_policer_command_fn (vlib_main_t * vm,
 385                          unformat_input_t * input, vlib_cli_command_t * cmd)
 386 {
 387   vnet_policer_main_t *pm = &vnet_policer_main;
 388   unformat_input_t _line_input, *line_input = &_line_input;
 389   u32 rx_sw_if_index;
 390   int rv;
 391   u8 *config_name = 0;
 392   int rx_set = 0;
 393   int is_add = 1;
 394   int is_show = 0;
 395   clib_error_t *error = NULL;
 396
 397   /* Get a line of input. */
 398   if (!unformat_user (input, unformat_line_input, line_input))
 399     return 0;
 400
 401   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 402     {
 403       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 404                     pm->vnet_main, &rx_sw_if_index))
 405         rx_set = 1;
 406       else if (unformat (line_input, "show"))
 407         is_show = 1;
 408       else if (unformat (line_input, "policer %s", &config_name))
 409         ;
 410       else if (unformat (line_input, "del"))
 411         is_add = 0;
 412       else
 413         break;
 414     }
 415
 416   if (rx_set == 0)
 417     {
 418       error = clib_error_return (0, "interface not set");
 419       goto done;
 420     }
 421
 422   if (is_show)
 423     {
 424       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 425       policer_read_response_type_st *policer;
 426       policer = pool_elt_at_index (pm->policers, pi);
 427
 428       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 429       goto done;
 430     }
 431
 432   if (is_add && config_name == 0)
 433     {
 434       error = clib_error_return (0, "policer config name required");
 435       goto done;
 436     }
 437
 438   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 439
 440   switch (rv)
 441     {
 442     case 0:
 443       break;
 444
 445     default:
 446       error = clib_error_return
 447         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 448       goto done;
 449     }
 450
 451 done:
 452   unformat_free (line_input);
 453
 454   return error;
 455 }
 456
 457 /* *INDENT-OFF* */
 458 VLIB_CLI_COMMAND (test_patch_command, static) = {
 459     .path = "test policer",
 460     .short_help =
 461     "intfc <intfc> policer <policer-config-name> [del]",
 462     .function = test_policer_command_fn,
 463 };
 464 /* *INDENT-ON* */
 465
 466 #endif /* TEST_CODE */
 467
 468
 469 typedef struct
 470 {
 471   u32 sw_if_index;
 472   u32 next_index;
 473   u32 table_index;
 474   u32 offset;
 475   u32 policer_index;
 476 } policer_classify_trace_t;
 477
 478 static u8 *
 479 format_policer_classify_trace (u8 * s, va_list * args)
 480 {
 481   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 482   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 483   policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
 484
 485   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 486               " policer_index %d",
 487               t->sw_if_index, t->next_index, t->table_index, t->offset,
 488               t->policer_index);
 489   return s;
 490 }
 491
 492 #define foreach_policer_classify_error                 \
 493 _(MISS, "Policer classify misses")                     \
 494 _(HIT, "Policer classify hits")                        \
 495 _(CHAIN_HIT, "Policer classify hits after chain walk") \
 496 _(DROP, "Policer classify action drop")
 497
 498 typedef enum
 499 {
 500 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 501   foreach_policer_classify_error
 502 #undef _
 503     POLICER_CLASSIFY_N_ERROR,
 504 } policer_classify_error_t;
 505
 506 static char *policer_classify_error_strings[] = {
 507 #define _(sym,string) string,
 508   foreach_policer_classify_error
 509 #undef _
 510 };
 511
 512 static inline uword
 513 policer_classify_inline (vlib_main_t * vm,
 514                          vlib_node_runtime_t * node,
 515                          vlib_frame_t * frame,
 516                          policer_classify_table_id_t tid)
 517 {
 518   u32 n_left_from, *from, *to_next;
 519   policer_classify_next_index_t next_index;
 520   policer_classify_main_t *pcm = &policer_classify_main;
 521   vnet_classify_main_t *vcm = pcm->vnet_classify_main;
 522   f64 now = vlib_time_now (vm);
 523   u32 hits = 0;
 524   u32 misses = 0;
 525   u32 chain_hits = 0;
 526   u32 n_next_nodes;
 527   u64 time_in_policer_periods;
 528
 529   time_in_policer_periods =
 530     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 531
 532   n_next_nodes = node->n_next_nodes;
 533
 534   from = vlib_frame_vector_args (frame);
 535   n_left_from = frame->n_vectors;
 536
 537   /* First pass: compute hashes */
 538   while (n_left_from > 2)
 539     {
 540       vlib_buffer_t *b0, *b1;
 541       u32 bi0, bi1;
 542       u8 *h0, *h1;
 543       u32 sw_if_index0, sw_if_index1;
 544       u32 table_index0, table_index1;
 545       vnet_classify_table_t *t0, *t1;
 546
 547       /* Prefetch next iteration */
 548       {
 549         vlib_buffer_t *p1, *p2;
 550
 551         p1 = vlib_get_buffer (vm, from[1]);
 552         p2 = vlib_get_buffer (vm, from[2]);
 553
 554         vlib_prefetch_buffer_header (p1, STORE);
 555         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 556         vlib_prefetch_buffer_header (p2, STORE);
 557         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 558       }
 559
 560       bi0 = from[0];
 561       b0 = vlib_get_buffer (vm, bi0);
 562       h0 = b0->data;
 563
 564       bi1 = from[1];
 565       b1 = vlib_get_buffer (vm, bi1);
 566       h1 = b1->data;
 567
 568       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 569       table_index0 =
 570         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 571
 572       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 573       table_index1 =
 574         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 575
 576       t0 = pool_elt_at_index (vcm->tables, table_index0);
 577
 578       t1 = pool_elt_at_index (vcm->tables, table_index1);
 579
 580       vnet_buffer (b0)->l2_classify.hash =
 581         vnet_classify_hash_packet (t0, (u8 *) h0);
 582
 583       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 584
 585       vnet_buffer (b1)->l2_classify.hash =
 586         vnet_classify_hash_packet (t1, (u8 *) h1);
 587
 588       vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
 589
 590       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 591
 592       vnet_buffer (b1)->l2_classify.table_index = table_index1;
 593
 594       from += 2;
 595       n_left_from -= 2;
 596     }
 597
 598   while (n_left_from > 0)
 599     {
 600       vlib_buffer_t *b0;
 601       u32 bi0;
 602       u8 *h0;
 603       u32 sw_if_index0;
 604       u32 table_index0;
 605       vnet_classify_table_t *t0;
 606
 607       bi0 = from[0];
 608       b0 = vlib_get_buffer (vm, bi0);
 609       h0 = b0->data;
 610
 611       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 612       table_index0 =
 613         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 614
 615       t0 = pool_elt_at_index (vcm->tables, table_index0);
 616       vnet_buffer (b0)->l2_classify.hash =
 617         vnet_classify_hash_packet (t0, (u8 *) h0);
 618
 619       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 620       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 621
 622       from++;
 623       n_left_from--;
 624     }
 625
 626   next_index = node->cached_next_index;
 627   from = vlib_frame_vector_args (frame);
 628   n_left_from = frame->n_vectors;
 629
 630   while (n_left_from > 0)
 631     {
 632       u32 n_left_to_next;
 633
 634       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 635
 636       /* Not enough load/store slots to dual loop... */
 637       while (n_left_from > 0 && n_left_to_next > 0)
 638         {
 639           u32 bi0;
 640           vlib_buffer_t *b0;
 641           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 642           u32 table_index0;
 643           vnet_classify_table_t *t0;
 644           vnet_classify_entry_t *e0;
 645           u64 hash0;
 646           u8 *h0;
 647           u8 act0;
 648
 649           /* Stride 3 seems to work best */
 650           if (PREDICT_TRUE (n_left_from > 3))
 651             {
 652               vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
 653               vnet_classify_table_t *tp1;
 654               u32 table_index1;
 655               u64 phash1;
 656
 657               table_index1 = vnet_buffer (p1)->l2_classify.table_index;
 658
 659               if (PREDICT_TRUE (table_index1 != ~0))
 660                 {
 661                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 662                   phash1 = vnet_buffer (p1)->l2_classify.hash;
 663                   vnet_classify_prefetch_entry (tp1, phash1);
 664                 }
 665             }
 666
 667           /* Speculatively enqueue b0 to the current next frame */
 668           bi0 = from[0];
 669           to_next[0] = bi0;
 670           from += 1;
 671           to_next += 1;
 672           n_left_from -= 1;
 673           n_left_to_next -= 1;
 674
 675           b0 = vlib_get_buffer (vm, bi0);
 676           h0 = b0->data;
 677           table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 678           e0 = 0;
 679           t0 = 0;
 680
 681           if (tid == POLICER_CLASSIFY_TABLE_L2)
 682             {
 683               /* Feature bitmap update and determine the next node */
 684               next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
 685                                             L2INPUT_FEAT_POLICER_CLAS);
 686             }
 687           else
 688             vnet_get_config_data (pcm->vnet_config_main[tid],
 689                                   &b0->current_config_index, &next0,
 690                                   /* # bytes of config data */ 0);
 691
 692           vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 693
 694           if (PREDICT_TRUE (table_index0 != ~0))
 695             {
 696               hash0 = vnet_buffer (b0)->l2_classify.hash;
 697               t0 = pool_elt_at_index (vcm->tables, table_index0);
 698               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 699
 700               if (e0)
 701                 {
 702                   act0 = vnet_policer_police (vm,
 703                                               b0,
 704                                               e0->next_index,
 705                                               time_in_policer_periods,
 706                                               e0->opaque_index);
 707                   if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 708                     {
 709                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 710                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 711                     }
 712                   hits++;
 713                 }
 714               else
 715                 {
 716                   while (1)
 717                     {
 718                       if (PREDICT_TRUE (t0->next_table_index != ~0))
 719                         {
 720                           t0 = pool_elt_at_index (vcm->tables,
 721                                                   t0->next_table_index);
 722                         }
 723                       else
 724                         {
 725                           next0 = (t0->miss_next_index < n_next_nodes) ?
 726                             t0->miss_next_index : next0;
 727                           misses++;
 728                           break;
 729                         }
 730
 731                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 732                       e0 =
 733                         vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 734                       if (e0)
 735                         {
 736                           act0 = vnet_policer_police (vm,
 737                                                       b0,
 738                                                       e0->next_index,
 739                                                       time_in_policer_periods,
 740                                                       e0->opaque_index);
 741                           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 742                             {
 743                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 744                               b0->error =
 745                                 node->errors[POLICER_CLASSIFY_ERROR_DROP];
 746                             }
 747                           hits++;
 748                           chain_hits++;
 749                           break;
 750                         }
 751                     }
 752                 }
 753             }
 754           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 755                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 756             {
 757               policer_classify_trace_t *t =
 758                 vlib_add_trace (vm, node, b0, sizeof (*t));
 759               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 760               t->next_index = next0;
 761               t->table_index = t0 ? t0 - vcm->tables : ~0;
 762               t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
 763               t->policer_index = e0 ? e0->next_index : ~0;
 764             }
 765
 766           /* Verify speculative enqueue, maybe switch current next frame */
 767           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 768                                            n_left_to_next, bi0, next0);
 769         }
 770
 771       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 772     }
 773
 774   vlib_node_increment_counter (vm, node->node_index,
 775                                POLICER_CLASSIFY_ERROR_MISS, misses);
 776   vlib_node_increment_counter (vm, node->node_index,
 777                                POLICER_CLASSIFY_ERROR_HIT, hits);
 778   vlib_node_increment_counter (vm, node->node_index,
 779                                POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
 780
 781   return frame->n_vectors;
 782 }
 783
 784 static uword
 785 ip4_policer_classify (vlib_main_t * vm,
 786                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 787 {
 788   return policer_classify_inline (vm, node, frame,
 789                                   POLICER_CLASSIFY_TABLE_IP4);
 790 }
 791
 792 /* *INDENT-OFF* */
 793 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 794   .function = ip4_policer_classify,
 795   .name = "ip4-policer-classify",
 796   .vector_size = sizeof (u32),
 797   .format_trace = format_policer_classify_trace,
 798   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 799   .error_strings = policer_classify_error_strings,
 800   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 801   .next_nodes = {
 802     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 803   },
 804 };
 805
 806 VLIB_NODE_FUNCTION_MULTIARCH (ip4_policer_classify_node, ip4_policer_classify);
 807 /* *INDENT-ON* */
 808
 809 static uword
 810 ip6_policer_classify (vlib_main_t * vm,
 811                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 812 {
 813   return policer_classify_inline (vm, node, frame,
 814                                   POLICER_CLASSIFY_TABLE_IP6);
 815 }
 816
 817 /* *INDENT-OFF* */
 818 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 819   .function = ip6_policer_classify,
 820   .name = "ip6-policer-classify",
 821   .vector_size = sizeof (u32),
 822   .format_trace = format_policer_classify_trace,
 823   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 824   .error_strings = policer_classify_error_strings,
 825   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 826   .next_nodes = {
 827     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 828   },
 829 };
 830
 831 VLIB_NODE_FUNCTION_MULTIARCH (ip6_policer_classify_node, ip6_policer_classify);
 832 /* *INDENT-ON* */
 833
 834 static uword
 835 l2_policer_classify (vlib_main_t * vm,
 836                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 837 {
 838   return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 839 }
 840
 841 /* *INDENT-OFF* */
 842 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
 843   .function = l2_policer_classify,
 844   .name = "l2-policer-classify",
 845   .vector_size = sizeof (u32),
 846   .format_trace = format_policer_classify_trace,
 847   .n_errors = ARRAY_LEN (policer_classify_error_strings),
 848   .error_strings = policer_classify_error_strings,
 849   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 850   .next_nodes = {
 851     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 852   },
 853 };
 854
 855 VLIB_NODE_FUNCTION_MULTIARCH (l2_policer_classify_node, l2_policer_classify);
 856 /* *INDENT-ON* */
 857
 858 static clib_error_t *
 859 policer_classify_init (vlib_main_t * vm)
 860 {
 861   policer_classify_main_t *pcm = &policer_classify_main;
 862
 863   pcm->vlib_main = vm;
 864   pcm->vnet_main = vnet_get_main ();
 865   pcm->vnet_classify_main = &vnet_classify_main;
 866
 867   /* Initialize L2 feature next-node indexes */
 868   feat_bitmap_init_next_nodes (vm,
 869                                l2_policer_classify_node.index,
 870                                L2INPUT_N_FEAT,
 871                                l2input_get_feat_names (),
 872                                pcm->feat_next_node_index);
 873
 874   return 0;
 875 }
 876
 877 VLIB_INIT_FUNCTION (policer_classify_init);
 878
 879 /*
 880  * fd.io coding-style-patch-verification: ON
 881  *
 882  * Local Variables:
 883  * eval: (c-set-style "gnu")
 884  * End:
 885  */