src/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/classify/policer_classify.h>
  23 #include <vnet/classify/vnet_classify.h>
  24
  25 #define IP4_NON_DSCP_BITS 0x03
  26 #define IP4_DSCP_SHIFT    2
  27 #define IP6_NON_DSCP_BITS 0xf03fffff
  28 #define IP6_DSCP_SHIFT    22
  29
  30 /* Dispatch functions meant to be instantiated elsewhere */
  31
  32 typedef struct
  33 {
  34   u32 next_index;
  35   u32 sw_if_index;
  36   u32 policer_index;
  37 } vnet_policer_trace_t;
  38
  39 /* packet trace format function */
  40 static u8 *
  41 format_policer_trace (u8 * s, va_list * args)
  42 {
  43   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  44   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  45   vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
  46
  47   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  48               t->sw_if_index, t->policer_index, t->next_index);
  49   return s;
  50 }
  51
  52 #define foreach_vnet_policer_error              \
  53 _(TRANSMIT, "Packets Transmitted")              \
  54 _(DROP, "Packets Dropped")
  55
  56 typedef enum
  57 {
  58 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  59   foreach_vnet_policer_error
  60 #undef _
  61     VNET_POLICER_N_ERROR,
  62 } vnet_policer_error_t;
  63
  64 static char *vnet_policer_error_strings[] = {
  65 #define _(sym,string) string,
  66   foreach_vnet_policer_error
  67 #undef _
  68 };
  69
  70 static_always_inline void
  71 vnet_policer_mark (vlib_buffer_t * b, u8 dscp)
  72 {
  73   ethernet_header_t *eh;
  74   ip4_header_t *ip4h;
  75   ip6_header_t *ip6h;
  76   u16 type;
  77
  78   eh = (ethernet_header_t *) b->data;
  79   type = clib_net_to_host_u16 (eh->type);
  80
  81   if (PREDICT_TRUE (type == ETHERNET_TYPE_IP4))
  82     {
  83       ip4h = (ip4_header_t *) & (b->data[sizeof (ethernet_header_t)]);;
  84       ip4h->tos &= IP4_NON_DSCP_BITS;
  85       ip4h->tos |= dscp << IP4_DSCP_SHIFT;
  86       ip4h->checksum = ip4_header_checksum (ip4h);
  87     }
  88   else
  89     {
  90       if (PREDICT_TRUE (type == ETHERNET_TYPE_IP6))
  91         {
  92           ip6h = (ip6_header_t *) & (b->data[sizeof (ethernet_header_t)]);
  93           ip6h->ip_version_traffic_class_and_flow_label &=
  94             clib_host_to_net_u32 (IP6_NON_DSCP_BITS);
  95           ip6h->ip_version_traffic_class_and_flow_label |=
  96             clib_host_to_net_u32 (dscp << IP6_DSCP_SHIFT);
  97         }
  98     }
  99 }
 100
 101 static_always_inline
 102   u8 vnet_policer_police (vlib_main_t * vm,
 103                           vlib_buffer_t * b,
 104                           u32 policer_index,
 105                           u64 time_in_policer_periods,
 106                           policer_result_e packet_color)
 107 {
 108   u8 act;
 109   u32 len;
 110   u32 col;
 111   policer_read_response_type_st *pol;
 112   vnet_policer_main_t *pm = &vnet_policer_main;
 113
 114   len = vlib_buffer_length_in_chain (vm, b);
 115   pol = &pm->policers[policer_index];
 116   col = vnet_police_packet (pol, len, packet_color, time_in_policer_periods);
 117   act = pol->action[col];
 118   if (PREDICT_TRUE (act == SSE2_QOS_ACTION_MARK_AND_TRANSMIT))
 119     vnet_policer_mark (b, pol->mark_dscp[col]);
 120
 121   return act;
 122 }
 123
 124 static inline uword
 125 vnet_policer_inline (vlib_main_t * vm,
 126                      vlib_node_runtime_t * node,
 127                      vlib_frame_t * frame, vnet_policer_index_t which)
 128 {
 129   u32 n_left_from, *from, *to_next;
 130   vnet_policer_next_t next_index;
 131   vnet_policer_main_t *pm = &vnet_policer_main;
 132   u64 time_in_policer_periods;
 133   u32 transmitted = 0;
 134
 135   time_in_policer_periods =
 136     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 137
 138   from = vlib_frame_vector_args (frame);
 139   n_left_from = frame->n_vectors;
 140   next_index = node->cached_next_index;
 141
 142   while (n_left_from > 0)
 143     {
 144       u32 n_left_to_next;
 145
 146       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 147
 148       while (n_left_from >= 4 && n_left_to_next >= 2)
 149         {
 150           u32 bi0, bi1;
 151           vlib_buffer_t *b0, *b1;
 152           u32 next0, next1;
 153           u32 sw_if_index0, sw_if_index1;
 154           u32 pi0 = 0, pi1 = 0;
 155           u8 act0, act1;
 156
 157           /* Prefetch next iteration. */
 158           {
 159             vlib_buffer_t *b2, *b3;
 160
 161             b2 = vlib_get_buffer (vm, from[2]);
 162             b3 = vlib_get_buffer (vm, from[3]);
 163
 164             vlib_prefetch_buffer_header (b2, LOAD);
 165             vlib_prefetch_buffer_header (b3, LOAD);
 166           }
 167
 168           /* speculatively enqueue b0 and b1 to the current next frame */
 169           to_next[0] = bi0 = from[0];
 170           to_next[1] = bi1 = from[1];
 171           from += 2;
 172           to_next += 2;
 173           n_left_from -= 2;
 174           n_left_to_next -= 2;
 175
 176           b0 = vlib_get_buffer (vm, bi0);
 177           b1 = vlib_get_buffer (vm, bi1);
 178
 179           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 180           next0 = VNET_POLICER_NEXT_TRANSMIT;
 181
 182           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 183           next1 = VNET_POLICER_NEXT_TRANSMIT;
 184
 185
 186           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 187             {
 188               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 189               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 190             }
 191
 192           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 193             {
 194               pi0 = vnet_buffer (b0)->policer.index;
 195               pi1 = vnet_buffer (b1)->policer.index;
 196             }
 197
 198           if (which == VNET_POLICER_INDEX_BY_EITHER)
 199             {
 200               pi0 = vnet_buffer (b0)->policer.index;
 201               pi0 = (pi0 != ~0) ? pi0 :
 202                 pm->policer_index_by_sw_if_index[sw_if_index0];
 203               pi1 = vnet_buffer (b1)->policer.index;
 204               pi1 = (pi1 != ~0) ? pi1 :
 205                 pm->policer_index_by_sw_if_index[sw_if_index1];
 206             }
 207
 208           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 209                                       POLICE_CONFORM /* no chaining */ );
 210
 211           act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
 212                                       POLICE_CONFORM /* no chaining */ );
 213
 214           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 215             {
 216               next0 = VNET_POLICER_NEXT_DROP;
 217               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 218             }
 219           else                  /* transmit or mark-and-transmit action */
 220             {
 221               transmitted++;
 222             }
 223
 224           if (PREDICT_FALSE (act1 == SSE2_QOS_ACTION_DROP))     /* drop action */
 225             {
 226               next1 = VNET_POLICER_NEXT_DROP;
 227               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 228             }
 229           else                  /* transmit or mark-and-transmit action */
 230             {
 231               transmitted++;
 232             }
 233
 234
 235           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
 236             {
 237               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 238                 {
 239                   vnet_policer_trace_t *t =
 240                     vlib_add_trace (vm, node, b0, sizeof (*t));
 241                   t->sw_if_index = sw_if_index0;
 242                   t->next_index = next0;
 243                 }
 244               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 245                 {
 246                   vnet_policer_trace_t *t =
 247                     vlib_add_trace (vm, node, b1, sizeof (*t));
 248                   t->sw_if_index = sw_if_index1;
 249                   t->next_index = next1;
 250                 }
 251             }
 252
 253           /* verify speculative enqueues, maybe switch current next frame */
 254           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 255                                            to_next, n_left_to_next,
 256                                            bi0, bi1, next0, next1);
 257         }
 258
 259       while (n_left_from > 0 && n_left_to_next > 0)
 260         {
 261           u32 bi0;
 262           vlib_buffer_t *b0;
 263           u32 next0;
 264           u32 sw_if_index0;
 265           u32 pi0 = 0;
 266           u8 act0;
 267
 268           bi0 = from[0];
 269           to_next[0] = bi0;
 270           from += 1;
 271           to_next += 1;
 272           n_left_from -= 1;
 273           n_left_to_next -= 1;
 274
 275           b0 = vlib_get_buffer (vm, bi0);
 276
 277           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 278           next0 = VNET_POLICER_NEXT_TRANSMIT;
 279
 280           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 281             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 282
 283           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 284             pi0 = vnet_buffer (b0)->policer.index;
 285
 286           if (which == VNET_POLICER_INDEX_BY_EITHER)
 287             {
 288               pi0 = vnet_buffer (b0)->policer.index;
 289               pi0 = (pi0 != ~0) ? pi0 :
 290                 pm->policer_index_by_sw_if_index[sw_if_index0];
 291             }
 292
 293           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 294                                       POLICE_CONFORM /* no chaining */ );
 295
 296           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 297             {
 298               next0 = VNET_POLICER_NEXT_DROP;
 299               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 300             }
 301           else                  /* transmit or mark-and-transmit action */
 302             {
 303               transmitted++;
 304             }
 305
 306           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 307                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 308             {
 309               vnet_policer_trace_t *t =
 310                 vlib_add_trace (vm, node, b0, sizeof (*t));
 311               t->sw_if_index = sw_if_index0;
 312               t->next_index = next0;
 313               t->policer_index = pi0;
 314             }
 315
 316           /* verify speculative enqueue, maybe switch current next frame */
 317           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 318                                            to_next, n_left_to_next,
 319                                            bi0, next0);
 320         }
 321
 322       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 323     }
 324
 325   vlib_node_increment_counter (vm, node->node_index,
 326                                VNET_POLICER_ERROR_TRANSMIT, transmitted);
 327   return frame->n_vectors;
 328 }
 329
 330 uword
 331 vnet_policer_by_sw_if_index (vlib_main_t * vm,
 332                              vlib_node_runtime_t * node, vlib_frame_t * frame)
 333 {
 334   return vnet_policer_inline (vm, node, frame,
 335                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 336 }
 337
 338 uword
 339 vnet_policer_by_opaque (vlib_main_t * vm,
 340                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 341 {
 342   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_OPAQUE);
 343 }
 344
 345 uword
 346 vnet_policer_by_either (vlib_main_t * vm,
 347                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 348 {
 349   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_EITHER);
 350 }
 351
 352 void
 353 vnet_policer_node_funcs_reference (void)
 354 {
 355 }
 356
 357
 358 #define TEST_CODE 1
 359
 360 #ifdef TEST_CODE
 361
 362 /* *INDENT-OFF* */
 363 VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = {
 364   .function = vnet_policer_by_sw_if_index,
 365   .name = "policer-by-sw-if-index",
 366   .vector_size = sizeof (u32),
 367   .format_trace = format_policer_trace,
 368   .type = VLIB_NODE_TYPE_INTERNAL,
 369
 370   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 371   .error_strings = vnet_policer_error_strings,
 372
 373   .n_next_nodes = VNET_POLICER_N_NEXT,
 374
 375   /* edit / add dispositions here */
 376   .next_nodes = {
 377     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 378     [VNET_POLICER_NEXT_DROP] = "error-drop",
 379   },
 380 };
 381
 382 VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
 383                               vnet_policer_by_sw_if_index);
 384 /* *INDENT-ON* */
 385
 386
 387 int
 388 test_policer_add_del (u32 rx_sw_if_index, u8 * config_name, int is_add)
 389 {
 390   vnet_policer_main_t *pm = &vnet_policer_main;
 391   policer_read_response_type_st *template;
 392   policer_read_response_type_st *policer;
 393   vnet_hw_interface_t *rxhi;
 394   uword *p;
 395
 396   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 397
 398   /* Make sure caller didn't pass a vlan subif, etc. */
 399   if (rxhi->sw_if_index != rx_sw_if_index)
 400     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 401
 402   if (is_add)
 403     {
 404
 405       p = hash_get_mem (pm->policer_config_by_name, config_name);
 406
 407       if (p == 0)
 408         return -2;
 409
 410       template = pool_elt_at_index (pm->policer_templates, p[0]);
 411
 412       vnet_hw_interface_rx_redirect_to_node
 413         (pm->vnet_main, rxhi->hw_if_index, policer_by_sw_if_index_node.index);
 414
 415       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 416
 417       policer[0] = template[0];
 418
 419       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 420       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 421         = policer - pm->policers;
 422     }
 423   else
 424     {
 425       u32 pi;
 426       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 427                                              rxhi->hw_if_index,
 428                                              ~0 /* disable */ );
 429
 430       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 431       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 432       pool_put_index (pm->policers, pi);
 433     }
 434
 435   return 0;
 436 }
 437
 438 static clib_error_t *
 439 test_policer_command_fn (vlib_main_t * vm,
 440                          unformat_input_t * input, vlib_cli_command_t * cmd)
 441 {
 442   vnet_policer_main_t *pm = &vnet_policer_main;
 443   unformat_input_t _line_input, *line_input = &_line_input;
 444   u32 rx_sw_if_index;
 445   int rv;
 446   u8 *config_name = 0;
 447   int rx_set = 0;
 448   int is_add = 1;
 449   int is_show = 0;
 450   clib_error_t *error = NULL;
 451
 452   /* Get a line of input. */
 453   if (!unformat_user (input, unformat_line_input, line_input))
 454     return 0;
 455
 456   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 457     {
 458       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 459                     pm->vnet_main, &rx_sw_if_index))
 460         rx_set = 1;
 461       else if (unformat (line_input, "show"))
 462         is_show = 1;
 463       else if (unformat (line_input, "policer %s", &config_name))
 464         ;
 465       else if (unformat (line_input, "del"))
 466         is_add = 0;
 467       else
 468         break;
 469     }
 470
 471   if (rx_set == 0)
 472     {
 473       error = clib_error_return (0, "interface not set");
 474       goto done;
 475     }
 476
 477   if (is_show)
 478     {
 479       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 480       policer_read_response_type_st *policer;
 481       policer = pool_elt_at_index (pm->policers, pi);
 482
 483       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 484       goto done;
 485     }
 486
 487   if (is_add && config_name == 0)
 488     {
 489       error = clib_error_return (0, "policer config name required");
 490       goto done;
 491     }
 492
 493   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 494
 495   switch (rv)
 496     {
 497     case 0:
 498       break;
 499
 500     default:
 501       error = clib_error_return
 502         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 503       goto done;
 504     }
 505
 506 done:
 507   unformat_free (line_input);
 508
 509   return error;
 510 }
 511
 512 /* *INDENT-OFF* */
 513 VLIB_CLI_COMMAND (test_patch_command, static) = {
 514     .path = "test policer",
 515     .short_help =
 516     "intfc <intfc> policer <policer-config-name> [del]",
 517     .function = test_policer_command_fn,
 518 };
 519 /* *INDENT-ON* */
 520
 521 #endif /* TEST_CODE */
 522
 523
 524 typedef struct
 525 {
 526   u32 sw_if_index;
 527   u32 next_index;
 528   u32 table_index;
 529   u32 offset;
 530   u32 policer_index;
 531 } policer_classify_trace_t;
 532
 533 static u8 *
 534 format_policer_classify_trace (u8 * s, va_list * args)
 535 {
 536   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 537   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 538   policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
 539
 540   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 541               " policer_index %d",
 542               t->sw_if_index, t->next_index, t->table_index, t->offset,
 543               t->policer_index);
 544   return s;
 545 }
 546
 547 #define foreach_policer_classify_error                 \
 548 _(MISS, "Policer classify misses")                     \
 549 _(HIT, "Policer classify hits")                        \
 550 _(CHAIN_HIT, "Polcier classify hits after chain walk") \
 551 _(DROP, "Policer classify action drop")
 552
 553 typedef enum
 554 {
 555 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 556   foreach_policer_classify_error
 557 #undef _
 558     POLICER_CLASSIFY_N_ERROR,
 559 } policer_classify_error_t;
 560
 561 static char *policer_classify_error_strings[] = {
 562 #define _(sym,string) string,
 563   foreach_policer_classify_error
 564 #undef _
 565 };
 566
 567 static inline uword
 568 policer_classify_inline (vlib_main_t * vm,
 569                          vlib_node_runtime_t * node,
 570                          vlib_frame_t * frame,
 571                          policer_classify_table_id_t tid)
 572 {
 573   u32 n_left_from, *from, *to_next;
 574   policer_classify_next_index_t next_index;
 575   policer_classify_main_t *pcm = &policer_classify_main;
 576   vnet_classify_main_t *vcm = pcm->vnet_classify_main;
 577   f64 now = vlib_time_now (vm);
 578   u32 hits = 0;
 579   u32 misses = 0;
 580   u32 chain_hits = 0;
 581   u32 drop = 0;
 582   u32 n_next_nodes;
 583   u64 time_in_policer_periods;
 584
 585   time_in_policer_periods =
 586     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 587
 588   n_next_nodes = node->n_next_nodes;
 589
 590   from = vlib_frame_vector_args (frame);
 591   n_left_from = frame->n_vectors;
 592
 593   /* First pass: compute hashes */
 594   while (n_left_from > 2)
 595     {
 596       vlib_buffer_t *b0, *b1;
 597       u32 bi0, bi1;
 598       u8 *h0, *h1;
 599       u32 sw_if_index0, sw_if_index1;
 600       u32 table_index0, table_index1;
 601       vnet_classify_table_t *t0, *t1;
 602
 603       /* Prefetch next iteration */
 604       {
 605         vlib_buffer_t *p1, *p2;
 606
 607         p1 = vlib_get_buffer (vm, from[1]);
 608         p2 = vlib_get_buffer (vm, from[2]);
 609
 610         vlib_prefetch_buffer_header (p1, STORE);
 611         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 612         vlib_prefetch_buffer_header (p2, STORE);
 613         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 614       }
 615
 616       bi0 = from[0];
 617       b0 = vlib_get_buffer (vm, bi0);
 618       h0 = b0->data;
 619
 620       bi1 = from[1];
 621       b1 = vlib_get_buffer (vm, bi1);
 622       h1 = b1->data;
 623
 624       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 625       table_index0 =
 626         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 627
 628       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 629       table_index1 =
 630         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 631
 632       t0 = pool_elt_at_index (vcm->tables, table_index0);
 633
 634       t1 = pool_elt_at_index (vcm->tables, table_index1);
 635
 636       vnet_buffer (b0)->l2_classify.hash =
 637         vnet_classify_hash_packet (t0, (u8 *) h0);
 638
 639       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 640
 641       vnet_buffer (b1)->l2_classify.hash =
 642         vnet_classify_hash_packet (t1, (u8 *) h1);
 643
 644       vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
 645
 646       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 647
 648       vnet_buffer (b1)->l2_classify.table_index = table_index1;
 649
 650       from += 2;
 651       n_left_from -= 2;
 652     }
 653
 654   while (n_left_from > 0)
 655     {
 656       vlib_buffer_t *b0;
 657       u32 bi0;
 658       u8 *h0;
 659       u32 sw_if_index0;
 660       u32 table_index0;
 661       vnet_classify_table_t *t0;
 662
 663       bi0 = from[0];
 664       b0 = vlib_get_buffer (vm, bi0);
 665       h0 = b0->data;
 666
 667       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 668       table_index0 =
 669         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 670
 671       t0 = pool_elt_at_index (vcm->tables, table_index0);
 672       vnet_buffer (b0)->l2_classify.hash =
 673         vnet_classify_hash_packet (t0, (u8 *) h0);
 674
 675       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 676       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 677
 678       from++;
 679       n_left_from--;
 680     }
 681
 682   next_index = node->cached_next_index;
 683   from = vlib_frame_vector_args (frame);
 684   n_left_from = frame->n_vectors;
 685
 686   while (n_left_from > 0)
 687     {
 688       u32 n_left_to_next;
 689
 690       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 691
 692       /* Not enough load/store slots to dual loop... */
 693       while (n_left_from > 0 && n_left_to_next > 0)
 694         {
 695           u32 bi0;
 696           vlib_buffer_t *b0;
 697           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 698           u32 table_index0;
 699           vnet_classify_table_t *t0;
 700           vnet_classify_entry_t *e0;
 701           u64 hash0;
 702           u8 *h0;
 703           u8 act0;
 704
 705           /* Stride 3 seems to work best */
 706           if (PREDICT_TRUE (n_left_from > 3))
 707             {
 708               vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
 709               vnet_classify_table_t *tp1;
 710               u32 table_index1;
 711               u64 phash1;
 712
 713               table_index1 = vnet_buffer (p1)->l2_classify.table_index;
 714
 715               if (PREDICT_TRUE (table_index1 != ~0))
 716                 {
 717                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 718                   phash1 = vnet_buffer (p1)->l2_classify.hash;
 719                   vnet_classify_prefetch_entry (tp1, phash1);
 720                 }
 721             }
 722
 723           /* Speculatively enqueue b0 to the current next frame */
 724           bi0 = from[0];
 725           to_next[0] = bi0;
 726           from += 1;
 727           to_next += 1;
 728           n_left_from -= 1;
 729           n_left_to_next -= 1;
 730
 731           b0 = vlib_get_buffer (vm, bi0);
 732           h0 = b0->data;
 733           table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 734           e0 = 0;
 735           t0 = 0;
 736
 737           if (tid == POLICER_CLASSIFY_TABLE_L2)
 738             {
 739               /* Feature bitmap update and determine the next node */
 740               next0 = vnet_l2_feature_next (b0, pcm->feat_next_node_index,
 741                                             L2INPUT_FEAT_POLICER_CLAS);
 742             }
 743           else
 744             vnet_get_config_data (pcm->vnet_config_main[tid],
 745                                   &b0->current_config_index, &next0,
 746                                   /* # bytes of config data */ 0);
 747
 748           vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 749
 750           if (PREDICT_TRUE (table_index0 != ~0))
 751             {
 752               hash0 = vnet_buffer (b0)->l2_classify.hash;
 753               t0 = pool_elt_at_index (vcm->tables, table_index0);
 754               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 755
 756               if (e0)
 757                 {
 758                   act0 = vnet_policer_police (vm,
 759                                               b0,
 760                                               e0->next_index,
 761                                               time_in_policer_periods,
 762                                               e0->opaque_index);
 763                   if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 764                     {
 765                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 766                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 767                       drop++;
 768                     }
 769                   hits++;
 770                 }
 771               else
 772                 {
 773                   while (1)
 774                     {
 775                       if (PREDICT_TRUE (t0->next_table_index != ~0))
 776                         {
 777                           t0 = pool_elt_at_index (vcm->tables,
 778                                                   t0->next_table_index);
 779                         }
 780                       else
 781                         {
 782                           next0 = (t0->miss_next_index < n_next_nodes) ?
 783                             t0->miss_next_index : next0;
 784                           misses++;
 785                           break;
 786                         }
 787
 788                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 789                       e0 =
 790                         vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 791                       if (e0)
 792                         {
 793                           act0 = vnet_policer_police (vm,
 794                                                       b0,
 795                                                       e0->next_index,
 796                                                       time_in_policer_periods,
 797                                                       e0->opaque_index);
 798                           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 799                             {
 800                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 801                               b0->error =
 802                                 node->errors[POLICER_CLASSIFY_ERROR_DROP];
 803                               drop++;
 804                             }
 805                           hits++;
 806                           chain_hits++;
 807                           break;
 808                         }
 809                     }
 810                 }
 811             }
 812           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 813                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 814             {
 815               policer_classify_trace_t *t =
 816                 vlib_add_trace (vm, node, b0, sizeof (*t));
 817               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 818               t->next_index = next0;
 819               t->table_index = t0 ? t0 - vcm->tables : ~0;
 820               t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
 821               t->policer_index = e0 ? e0->next_index : ~0;
 822             }
 823
 824           /* Verify speculative enqueue, maybe switch current next frame */
 825           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 826                                            n_left_to_next, bi0, next0);
 827         }
 828
 829       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 830     }
 831
 832   vlib_node_increment_counter (vm, node->node_index,
 833                                POLICER_CLASSIFY_ERROR_MISS, misses);
 834   vlib_node_increment_counter (vm, node->node_index,
 835                                POLICER_CLASSIFY_ERROR_HIT, hits);
 836   vlib_node_increment_counter (vm, node->node_index,
 837                                POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
 838   vlib_node_increment_counter (vm, node->node_index,
 839                                POLICER_CLASSIFY_ERROR_DROP, drop);
 840
 841   return frame->n_vectors;
 842 }
 843
 844 static uword
 845 ip4_policer_classify (vlib_main_t * vm,
 846                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 847 {
 848   return policer_classify_inline (vm, node, frame,
 849                                   POLICER_CLASSIFY_TABLE_IP4);
 850 }
 851
 852 /* *INDENT-OFF* */
 853 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 854   .function = ip4_policer_classify,
 855   .name = "ip4-policer-classify",
 856   .vector_size = sizeof (u32),
 857   .format_trace = format_policer_classify_trace,
 858   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 859   .error_strings = policer_classify_error_strings,
 860   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 861   .next_nodes = {
 862     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 863   },
 864 };
 865
 866 VLIB_NODE_FUNCTION_MULTIARCH (ip4_policer_classify_node, ip4_policer_classify);
 867 /* *INDENT-ON* */
 868
 869 static uword
 870 ip6_policer_classify (vlib_main_t * vm,
 871                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 872 {
 873   return policer_classify_inline (vm, node, frame,
 874                                   POLICER_CLASSIFY_TABLE_IP6);
 875 }
 876
 877 /* *INDENT-OFF* */
 878 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 879   .function = ip6_policer_classify,
 880   .name = "ip6-policer-classify",
 881   .vector_size = sizeof (u32),
 882   .format_trace = format_policer_classify_trace,
 883   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 884   .error_strings = policer_classify_error_strings,
 885   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 886   .next_nodes = {
 887     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 888   },
 889 };
 890
 891 VLIB_NODE_FUNCTION_MULTIARCH (ip6_policer_classify_node, ip6_policer_classify);
 892 /* *INDENT-ON* */
 893
 894 static uword
 895 l2_policer_classify (vlib_main_t * vm,
 896                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 897 {
 898   return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 899 }
 900
 901 VLIB_REGISTER_NODE (l2_policer_classify_node) =
 902 {
 903   .function = l2_policer_classify,.name = "l2-policer-classify",.vector_size =
 904     sizeof (u32),.format_trace = format_policer_classify_trace,.n_errors =
 905     ARRAY_LEN (policer_classify_error_strings),.error_strings =
 906     policer_classify_error_strings,.n_next_nodes =
 907     POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,.next_nodes =
 908   {
 909   [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",}
 910 ,};
 911
 912 VLIB_NODE_FUNCTION_MULTIARCH (l2_policer_classify_node, l2_policer_classify);
 913
 914
 915 static clib_error_t *
 916 policer_classify_init (vlib_main_t * vm)
 917 {
 918   policer_classify_main_t *pcm = &policer_classify_main;
 919
 920   pcm->vlib_main = vm;
 921   pcm->vnet_main = vnet_get_main ();
 922   pcm->vnet_classify_main = &vnet_classify_main;
 923
 924   /* Initialize L2 feature next-node indexes */
 925   feat_bitmap_init_next_nodes (vm,
 926                                l2_policer_classify_node.index,
 927                                L2INPUT_N_FEAT,
 928                                l2input_get_feat_names (),
 929                                pcm->feat_next_node_index);
 930
 931   return 0;
 932 }
 933
 934 VLIB_INIT_FUNCTION (policer_classify_init);
 935
 936 /*
 937  * fd.io coding-style-patch-verification: ON
 938  *
 939  * Local Variables:
 940  * eval: (c-set-style "gnu")
 941  * End:
 942  */