src/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/policer/police_inlines.h>
  22 #include <vnet/ip/ip.h>
  23 #include <vnet/classify/policer_classify.h>
  24 #include <vnet/classify/vnet_classify.h>
  25
  26
  27 /* Dispatch functions meant to be instantiated elsewhere */
  28
  29 typedef struct
  30 {
  31   u32 next_index;
  32   u32 sw_if_index;
  33   u32 policer_index;
  34 } vnet_policer_trace_t;
  35
  36 /* packet trace format function */
  37 static u8 *
  38 format_policer_trace (u8 * s, va_list * args)
  39 {
  40   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  41   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  42   vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
  43
  44   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  45               t->sw_if_index, t->policer_index, t->next_index);
  46   return s;
  47 }
  48
  49 #define foreach_vnet_policer_error              \
  50 _(TRANSMIT, "Packets Transmitted")              \
  51 _(DROP, "Packets Dropped")
  52
  53 typedef enum
  54 {
  55 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  56   foreach_vnet_policer_error
  57 #undef _
  58     VNET_POLICER_N_ERROR,
  59 } vnet_policer_error_t;
  60
  61 static char *vnet_policer_error_strings[] = {
  62 #define _(sym,string) string,
  63   foreach_vnet_policer_error
  64 #undef _
  65 };
  66
  67 static inline uword
  68 vnet_policer_inline (vlib_main_t * vm,
  69                      vlib_node_runtime_t * node,
  70                      vlib_frame_t * frame, vnet_policer_index_t which)
  71 {
  72   u32 n_left_from, *from, *to_next;
  73   vnet_policer_next_t next_index;
  74   vnet_policer_main_t *pm = &vnet_policer_main;
  75   u64 time_in_policer_periods;
  76   u32 transmitted = 0;
  77
  78   time_in_policer_periods =
  79     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
  80
  81   from = vlib_frame_vector_args (frame);
  82   n_left_from = frame->n_vectors;
  83   next_index = node->cached_next_index;
  84
  85   while (n_left_from > 0)
  86     {
  87       u32 n_left_to_next;
  88
  89       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
  90
  91       while (n_left_from >= 4 && n_left_to_next >= 2)
  92         {
  93           u32 bi0, bi1;
  94           vlib_buffer_t *b0, *b1;
  95           u32 next0, next1;
  96           u32 sw_if_index0, sw_if_index1;
  97           u32 pi0 = 0, pi1 = 0;
  98           u8 act0, act1;
  99
 100           /* Prefetch next iteration. */
 101           {
 102             vlib_buffer_t *b2, *b3;
 103
 104             b2 = vlib_get_buffer (vm, from[2]);
 105             b3 = vlib_get_buffer (vm, from[3]);
 106
 107             vlib_prefetch_buffer_header (b2, LOAD);
 108             vlib_prefetch_buffer_header (b3, LOAD);
 109           }
 110
 111           /* speculatively enqueue b0 and b1 to the current next frame */
 112           to_next[0] = bi0 = from[0];
 113           to_next[1] = bi1 = from[1];
 114           from += 2;
 115           to_next += 2;
 116           n_left_from -= 2;
 117           n_left_to_next -= 2;
 118
 119           b0 = vlib_get_buffer (vm, bi0);
 120           b1 = vlib_get_buffer (vm, bi1);
 121
 122           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 123           next0 = VNET_POLICER_NEXT_TRANSMIT;
 124
 125           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 126           next1 = VNET_POLICER_NEXT_TRANSMIT;
 127
 128
 129           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 130             {
 131               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 132               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 133             }
 134
 135           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 136             {
 137               pi0 = vnet_buffer (b0)->policer.index;
 138               pi1 = vnet_buffer (b1)->policer.index;
 139             }
 140
 141           if (which == VNET_POLICER_INDEX_BY_EITHER)
 142             {
 143               pi0 = vnet_buffer (b0)->policer.index;
 144               pi0 = (pi0 != ~0) ? pi0 :
 145                 pm->policer_index_by_sw_if_index[sw_if_index0];
 146               pi1 = vnet_buffer (b1)->policer.index;
 147               pi1 = (pi1 != ~0) ? pi1 :
 148                 pm->policer_index_by_sw_if_index[sw_if_index1];
 149             }
 150
 151           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 152                                       POLICE_CONFORM /* no chaining */ );
 153
 154           act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
 155                                       POLICE_CONFORM /* no chaining */ );
 156
 157           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 158             {
 159               next0 = VNET_POLICER_NEXT_DROP;
 160               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 161             }
 162           else                  /* transmit or mark-and-transmit action */
 163             {
 164               transmitted++;
 165             }
 166
 167           if (PREDICT_FALSE (act1 == SSE2_QOS_ACTION_DROP))     /* drop action */
 168             {
 169               next1 = VNET_POLICER_NEXT_DROP;
 170               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 171             }
 172           else                  /* transmit or mark-and-transmit action */
 173             {
 174               transmitted++;
 175             }
 176
 177
 178           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
 179             {
 180               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 181                 {
 182                   vnet_policer_trace_t *t =
 183                     vlib_add_trace (vm, node, b0, sizeof (*t));
 184                   t->sw_if_index = sw_if_index0;
 185                   t->next_index = next0;
 186                 }
 187               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 188                 {
 189                   vnet_policer_trace_t *t =
 190                     vlib_add_trace (vm, node, b1, sizeof (*t));
 191                   t->sw_if_index = sw_if_index1;
 192                   t->next_index = next1;
 193                 }
 194             }
 195
 196           /* verify speculative enqueues, maybe switch current next frame */
 197           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 198                                            to_next, n_left_to_next,
 199                                            bi0, bi1, next0, next1);
 200         }
 201
 202       while (n_left_from > 0 && n_left_to_next > 0)
 203         {
 204           u32 bi0;
 205           vlib_buffer_t *b0;
 206           u32 next0;
 207           u32 sw_if_index0;
 208           u32 pi0 = 0;
 209           u8 act0;
 210
 211           bi0 = from[0];
 212           to_next[0] = bi0;
 213           from += 1;
 214           to_next += 1;
 215           n_left_from -= 1;
 216           n_left_to_next -= 1;
 217
 218           b0 = vlib_get_buffer (vm, bi0);
 219
 220           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 221           next0 = VNET_POLICER_NEXT_TRANSMIT;
 222
 223           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 224             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 225
 226           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 227             pi0 = vnet_buffer (b0)->policer.index;
 228
 229           if (which == VNET_POLICER_INDEX_BY_EITHER)
 230             {
 231               pi0 = vnet_buffer (b0)->policer.index;
 232               pi0 = (pi0 != ~0) ? pi0 :
 233                 pm->policer_index_by_sw_if_index[sw_if_index0];
 234             }
 235
 236           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 237                                       POLICE_CONFORM /* no chaining */ );
 238
 239           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 240             {
 241               next0 = VNET_POLICER_NEXT_DROP;
 242               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 243             }
 244           else                  /* transmit or mark-and-transmit action */
 245             {
 246               transmitted++;
 247             }
 248
 249           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 250                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 251             {
 252               vnet_policer_trace_t *t =
 253                 vlib_add_trace (vm, node, b0, sizeof (*t));
 254               t->sw_if_index = sw_if_index0;
 255               t->next_index = next0;
 256               t->policer_index = pi0;
 257             }
 258
 259           /* verify speculative enqueue, maybe switch current next frame */
 260           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 261                                            to_next, n_left_to_next,
 262                                            bi0, next0);
 263         }
 264
 265       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 266     }
 267
 268   vlib_node_increment_counter (vm, node->node_index,
 269                                VNET_POLICER_ERROR_TRANSMIT, transmitted);
 270   return frame->n_vectors;
 271 }
 272
 273 uword
 274 vnet_policer_by_sw_if_index (vlib_main_t * vm,
 275                              vlib_node_runtime_t * node, vlib_frame_t * frame)
 276 {
 277   return vnet_policer_inline (vm, node, frame,
 278                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 279 }
 280
 281 uword
 282 vnet_policer_by_opaque (vlib_main_t * vm,
 283                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 284 {
 285   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_OPAQUE);
 286 }
 287
 288 uword
 289 vnet_policer_by_either (vlib_main_t * vm,
 290                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 291 {
 292   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_EITHER);
 293 }
 294
 295 void
 296 vnet_policer_node_funcs_reference (void)
 297 {
 298 }
 299
 300
 301 #define TEST_CODE 1
 302
 303 #ifdef TEST_CODE
 304
 305 /* *INDENT-OFF* */
 306 VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = {
 307   .function = vnet_policer_by_sw_if_index,
 308   .name = "policer-by-sw-if-index",
 309   .vector_size = sizeof (u32),
 310   .format_trace = format_policer_trace,
 311   .type = VLIB_NODE_TYPE_INTERNAL,
 312
 313   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 314   .error_strings = vnet_policer_error_strings,
 315
 316   .n_next_nodes = VNET_POLICER_N_NEXT,
 317
 318   /* edit / add dispositions here */
 319   .next_nodes = {
 320     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 321     [VNET_POLICER_NEXT_DROP] = "error-drop",
 322   },
 323 };
 324
 325 VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
 326                               vnet_policer_by_sw_if_index);
 327 /* *INDENT-ON* */
 328
 329
 330 int
 331 test_policer_add_del (u32 rx_sw_if_index, u8 * config_name, int is_add)
 332 {
 333   vnet_policer_main_t *pm = &vnet_policer_main;
 334   policer_read_response_type_st *template;
 335   policer_read_response_type_st *policer;
 336   vnet_hw_interface_t *rxhi;
 337   uword *p;
 338
 339   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 340
 341   /* Make sure caller didn't pass a vlan subif, etc. */
 342   if (rxhi->sw_if_index != rx_sw_if_index)
 343     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 344
 345   if (is_add)
 346     {
 347
 348       p = hash_get_mem (pm->policer_config_by_name, config_name);
 349
 350       if (p == 0)
 351         return -2;
 352
 353       template = pool_elt_at_index (pm->policer_templates, p[0]);
 354
 355       vnet_hw_interface_rx_redirect_to_node
 356         (pm->vnet_main, rxhi->hw_if_index, policer_by_sw_if_index_node.index);
 357
 358       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 359
 360       policer[0] = template[0];
 361
 362       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 363       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 364         = policer - pm->policers;
 365     }
 366   else
 367     {
 368       u32 pi;
 369       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 370                                              rxhi->hw_if_index,
 371                                              ~0 /* disable */ );
 372
 373       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 374       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 375       pool_put_index (pm->policers, pi);
 376     }
 377
 378   return 0;
 379 }
 380
 381 static clib_error_t *
 382 test_policer_command_fn (vlib_main_t * vm,
 383                          unformat_input_t * input, vlib_cli_command_t * cmd)
 384 {
 385   vnet_policer_main_t *pm = &vnet_policer_main;
 386   unformat_input_t _line_input, *line_input = &_line_input;
 387   u32 rx_sw_if_index;
 388   int rv;
 389   u8 *config_name = 0;
 390   int rx_set = 0;
 391   int is_add = 1;
 392   int is_show = 0;
 393   clib_error_t *error = NULL;
 394
 395   /* Get a line of input. */
 396   if (!unformat_user (input, unformat_line_input, line_input))
 397     return 0;
 398
 399   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 400     {
 401       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 402                     pm->vnet_main, &rx_sw_if_index))
 403         rx_set = 1;
 404       else if (unformat (line_input, "show"))
 405         is_show = 1;
 406       else if (unformat (line_input, "policer %s", &config_name))
 407         ;
 408       else if (unformat (line_input, "del"))
 409         is_add = 0;
 410       else
 411         break;
 412     }
 413
 414   if (rx_set == 0)
 415     {
 416       error = clib_error_return (0, "interface not set");
 417       goto done;
 418     }
 419
 420   if (is_show)
 421     {
 422       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 423       policer_read_response_type_st *policer;
 424       policer = pool_elt_at_index (pm->policers, pi);
 425
 426       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 427       goto done;
 428     }
 429
 430   if (is_add && config_name == 0)
 431     {
 432       error = clib_error_return (0, "policer config name required");
 433       goto done;
 434     }
 435
 436   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 437
 438   switch (rv)
 439     {
 440     case 0:
 441       break;
 442
 443     default:
 444       error = clib_error_return
 445         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 446       goto done;
 447     }
 448
 449 done:
 450   unformat_free (line_input);
 451
 452   return error;
 453 }
 454
 455 /* *INDENT-OFF* */
 456 VLIB_CLI_COMMAND (test_patch_command, static) = {
 457     .path = "test policer",
 458     .short_help =
 459     "intfc <intfc> policer <policer-config-name> [del]",
 460     .function = test_policer_command_fn,
 461 };
 462 /* *INDENT-ON* */
 463
 464 #endif /* TEST_CODE */
 465
 466
 467 typedef struct
 468 {
 469   u32 sw_if_index;
 470   u32 next_index;
 471   u32 table_index;
 472   u32 offset;
 473   u32 policer_index;
 474 } policer_classify_trace_t;
 475
 476 static u8 *
 477 format_policer_classify_trace (u8 * s, va_list * args)
 478 {
 479   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 480   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 481   policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
 482
 483   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 484               " policer_index %d",
 485               t->sw_if_index, t->next_index, t->table_index, t->offset,
 486               t->policer_index);
 487   return s;
 488 }
 489
 490 #define foreach_policer_classify_error                 \
 491 _(MISS, "Policer classify misses")                     \
 492 _(HIT, "Policer classify hits")                        \
 493 _(CHAIN_HIT, "Polcier classify hits after chain walk") \
 494 _(DROP, "Policer classify action drop")
 495
 496 typedef enum
 497 {
 498 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 499   foreach_policer_classify_error
 500 #undef _
 501     POLICER_CLASSIFY_N_ERROR,
 502 } policer_classify_error_t;
 503
 504 static char *policer_classify_error_strings[] = {
 505 #define _(sym,string) string,
 506   foreach_policer_classify_error
 507 #undef _
 508 };
 509
 510 static inline uword
 511 policer_classify_inline (vlib_main_t * vm,
 512                          vlib_node_runtime_t * node,
 513                          vlib_frame_t * frame,
 514                          policer_classify_table_id_t tid)
 515 {
 516   u32 n_left_from, *from, *to_next;
 517   policer_classify_next_index_t next_index;
 518   policer_classify_main_t *pcm = &policer_classify_main;
 519   vnet_classify_main_t *vcm = pcm->vnet_classify_main;
 520   f64 now = vlib_time_now (vm);
 521   u32 hits = 0;
 522   u32 misses = 0;
 523   u32 chain_hits = 0;
 524   u32 drop = 0;
 525   u32 n_next_nodes;
 526   u64 time_in_policer_periods;
 527
 528   time_in_policer_periods =
 529     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 530
 531   n_next_nodes = node->n_next_nodes;
 532
 533   from = vlib_frame_vector_args (frame);
 534   n_left_from = frame->n_vectors;
 535
 536   /* First pass: compute hashes */
 537   while (n_left_from > 2)
 538     {
 539       vlib_buffer_t *b0, *b1;
 540       u32 bi0, bi1;
 541       u8 *h0, *h1;
 542       u32 sw_if_index0, sw_if_index1;
 543       u32 table_index0, table_index1;
 544       vnet_classify_table_t *t0, *t1;
 545
 546       /* Prefetch next iteration */
 547       {
 548         vlib_buffer_t *p1, *p2;
 549
 550         p1 = vlib_get_buffer (vm, from[1]);
 551         p2 = vlib_get_buffer (vm, from[2]);
 552
 553         vlib_prefetch_buffer_header (p1, STORE);
 554         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 555         vlib_prefetch_buffer_header (p2, STORE);
 556         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 557       }
 558
 559       bi0 = from[0];
 560       b0 = vlib_get_buffer (vm, bi0);
 561       h0 = b0->data;
 562
 563       bi1 = from[1];
 564       b1 = vlib_get_buffer (vm, bi1);
 565       h1 = b1->data;
 566
 567       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 568       table_index0 =
 569         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 570
 571       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 572       table_index1 =
 573         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 574
 575       t0 = pool_elt_at_index (vcm->tables, table_index0);
 576
 577       t1 = pool_elt_at_index (vcm->tables, table_index1);
 578
 579       vnet_buffer (b0)->l2_classify.hash =
 580         vnet_classify_hash_packet (t0, (u8 *) h0);
 581
 582       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 583
 584       vnet_buffer (b1)->l2_classify.hash =
 585         vnet_classify_hash_packet (t1, (u8 *) h1);
 586
 587       vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
 588
 589       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 590
 591       vnet_buffer (b1)->l2_classify.table_index = table_index1;
 592
 593       from += 2;
 594       n_left_from -= 2;
 595     }
 596
 597   while (n_left_from > 0)
 598     {
 599       vlib_buffer_t *b0;
 600       u32 bi0;
 601       u8 *h0;
 602       u32 sw_if_index0;
 603       u32 table_index0;
 604       vnet_classify_table_t *t0;
 605
 606       bi0 = from[0];
 607       b0 = vlib_get_buffer (vm, bi0);
 608       h0 = b0->data;
 609
 610       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 611       table_index0 =
 612         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 613
 614       t0 = pool_elt_at_index (vcm->tables, table_index0);
 615       vnet_buffer (b0)->l2_classify.hash =
 616         vnet_classify_hash_packet (t0, (u8 *) h0);
 617
 618       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 619       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 620
 621       from++;
 622       n_left_from--;
 623     }
 624
 625   next_index = node->cached_next_index;
 626   from = vlib_frame_vector_args (frame);
 627   n_left_from = frame->n_vectors;
 628
 629   while (n_left_from > 0)
 630     {
 631       u32 n_left_to_next;
 632
 633       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 634
 635       /* Not enough load/store slots to dual loop... */
 636       while (n_left_from > 0 && n_left_to_next > 0)
 637         {
 638           u32 bi0;
 639           vlib_buffer_t *b0;
 640           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 641           u32 table_index0;
 642           vnet_classify_table_t *t0;
 643           vnet_classify_entry_t *e0;
 644           u64 hash0;
 645           u8 *h0;
 646           u8 act0;
 647
 648           /* Stride 3 seems to work best */
 649           if (PREDICT_TRUE (n_left_from > 3))
 650             {
 651               vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
 652               vnet_classify_table_t *tp1;
 653               u32 table_index1;
 654               u64 phash1;
 655
 656               table_index1 = vnet_buffer (p1)->l2_classify.table_index;
 657
 658               if (PREDICT_TRUE (table_index1 != ~0))
 659                 {
 660                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 661                   phash1 = vnet_buffer (p1)->l2_classify.hash;
 662                   vnet_classify_prefetch_entry (tp1, phash1);
 663                 }
 664             }
 665
 666           /* Speculatively enqueue b0 to the current next frame */
 667           bi0 = from[0];
 668           to_next[0] = bi0;
 669           from += 1;
 670           to_next += 1;
 671           n_left_from -= 1;
 672           n_left_to_next -= 1;
 673
 674           b0 = vlib_get_buffer (vm, bi0);
 675           h0 = b0->data;
 676           table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 677           e0 = 0;
 678           t0 = 0;
 679
 680           if (tid == POLICER_CLASSIFY_TABLE_L2)
 681             {
 682               /* Feature bitmap update and determine the next node */
 683               next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
 684                                             L2INPUT_FEAT_POLICER_CLAS);
 685             }
 686           else
 687             vnet_get_config_data (pcm->vnet_config_main[tid],
 688                                   &b0->current_config_index, &next0,
 689                                   /* # bytes of config data */ 0);
 690
 691           vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 692
 693           if (PREDICT_TRUE (table_index0 != ~0))
 694             {
 695               hash0 = vnet_buffer (b0)->l2_classify.hash;
 696               t0 = pool_elt_at_index (vcm->tables, table_index0);
 697               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 698
 699               if (e0)
 700                 {
 701                   act0 = vnet_policer_police (vm,
 702                                               b0,
 703                                               e0->next_index,
 704                                               time_in_policer_periods,
 705                                               e0->opaque_index);
 706                   if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 707                     {
 708                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 709                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 710                       drop++;
 711                     }
 712                   hits++;
 713                 }
 714               else
 715                 {
 716                   while (1)
 717                     {
 718                       if (PREDICT_TRUE (t0->next_table_index != ~0))
 719                         {
 720                           t0 = pool_elt_at_index (vcm->tables,
 721                                                   t0->next_table_index);
 722                         }
 723                       else
 724                         {
 725                           next0 = (t0->miss_next_index < n_next_nodes) ?
 726                             t0->miss_next_index : next0;
 727                           misses++;
 728                           break;
 729                         }
 730
 731                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 732                       e0 =
 733                         vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 734                       if (e0)
 735                         {
 736                           act0 = vnet_policer_police (vm,
 737                                                       b0,
 738                                                       e0->next_index,
 739                                                       time_in_policer_periods,
 740                                                       e0->opaque_index);
 741                           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 742                             {
 743                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 744                               b0->error =
 745                                 node->errors[POLICER_CLASSIFY_ERROR_DROP];
 746                               drop++;
 747                             }
 748                           hits++;
 749                           chain_hits++;
 750                           break;
 751                         }
 752                     }
 753                 }
 754             }
 755           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 756                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 757             {
 758               policer_classify_trace_t *t =
 759                 vlib_add_trace (vm, node, b0, sizeof (*t));
 760               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 761               t->next_index = next0;
 762               t->table_index = t0 ? t0 - vcm->tables : ~0;
 763               t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
 764               t->policer_index = e0 ? e0->next_index : ~0;
 765             }
 766
 767           /* Verify speculative enqueue, maybe switch current next frame */
 768           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 769                                            n_left_to_next, bi0, next0);
 770         }
 771
 772       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 773     }
 774
 775   vlib_node_increment_counter (vm, node->node_index,
 776                                POLICER_CLASSIFY_ERROR_MISS, misses);
 777   vlib_node_increment_counter (vm, node->node_index,
 778                                POLICER_CLASSIFY_ERROR_HIT, hits);
 779   vlib_node_increment_counter (vm, node->node_index,
 780                                POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
 781   vlib_node_increment_counter (vm, node->node_index,
 782                                POLICER_CLASSIFY_ERROR_DROP, drop);
 783
 784   return frame->n_vectors;
 785 }
 786
 787 static uword
 788 ip4_policer_classify (vlib_main_t * vm,
 789                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 790 {
 791   return policer_classify_inline (vm, node, frame,
 792                                   POLICER_CLASSIFY_TABLE_IP4);
 793 }
 794
 795 /* *INDENT-OFF* */
 796 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 797   .function = ip4_policer_classify,
 798   .name = "ip4-policer-classify",
 799   .vector_size = sizeof (u32),
 800   .format_trace = format_policer_classify_trace,
 801   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 802   .error_strings = policer_classify_error_strings,
 803   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 804   .next_nodes = {
 805     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 806   },
 807 };
 808
 809 VLIB_NODE_FUNCTION_MULTIARCH (ip4_policer_classify_node, ip4_policer_classify);
 810 /* *INDENT-ON* */
 811
 812 static uword
 813 ip6_policer_classify (vlib_main_t * vm,
 814                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 815 {
 816   return policer_classify_inline (vm, node, frame,
 817                                   POLICER_CLASSIFY_TABLE_IP6);
 818 }
 819
 820 /* *INDENT-OFF* */
 821 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 822   .function = ip6_policer_classify,
 823   .name = "ip6-policer-classify",
 824   .vector_size = sizeof (u32),
 825   .format_trace = format_policer_classify_trace,
 826   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 827   .error_strings = policer_classify_error_strings,
 828   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 829   .next_nodes = {
 830     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 831   },
 832 };
 833
 834 VLIB_NODE_FUNCTION_MULTIARCH (ip6_policer_classify_node, ip6_policer_classify);
 835 /* *INDENT-ON* */
 836
 837 static uword
 838 l2_policer_classify (vlib_main_t * vm,
 839                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 840 {
 841   return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 842 }
 843
 844 VLIB_REGISTER_NODE (l2_policer_classify_node) =
 845 {
 846   .function = l2_policer_classify,.name = "l2-policer-classify",.vector_size =
 847     sizeof (u32),.format_trace = format_policer_classify_trace,.n_errors =
 848     ARRAY_LEN (policer_classify_error_strings),.error_strings =
 849     policer_classify_error_strings,.n_next_nodes =
 850     POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,.next_nodes =
 851   {
 852   [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",}
 853 ,};
 854
 855 VLIB_NODE_FUNCTION_MULTIARCH (l2_policer_classify_node, l2_policer_classify);
 856
 857
 858 static clib_error_t *
 859 policer_classify_init (vlib_main_t * vm)
 860 {
 861   policer_classify_main_t *pcm = &policer_classify_main;
 862
 863   pcm->vlib_main = vm;
 864   pcm->vnet_main = vnet_get_main ();
 865   pcm->vnet_classify_main = &vnet_classify_main;
 866
 867   /* Initialize L2 feature next-node indexes */
 868   feat_bitmap_init_next_nodes (vm,
 869                                l2_policer_classify_node.index,
 870                                L2INPUT_N_FEAT,
 871                                l2input_get_feat_names (),
 872                                pcm->feat_next_node_index);
 873
 874   return 0;
 875 }
 876
 877 VLIB_INIT_FUNCTION (policer_classify_init);
 878
 879 /*
 880  * fd.io coding-style-patch-verification: ON
 881  *
 882  * Local Variables:
 883  * eval: (c-set-style "gnu")
 884  * End:
 885  */