vnet/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/classify/policer_classify.h>
  23 #include <vnet/classify/vnet_classify.h>
  24
  25 #define IP4_NON_DSCP_BITS 0x03
  26 #define IP4_DSCP_SHIFT    2
  27 #define IP6_NON_DSCP_BITS 0xf03fffff
  28 #define IP6_DSCP_SHIFT    22
  29
  30 /* Dispatch functions meant to be instantiated elsewhere */
  31
  32 typedef struct
  33 {
  34   u32 next_index;
  35   u32 sw_if_index;
  36   u32 policer_index;
  37 } vnet_policer_trace_t;
  38
  39 /* packet trace format function */
  40 static u8 *
  41 format_policer_trace (u8 * s, va_list * args)
  42 {
  43   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  44   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  45   vnet_policer_trace_t *t = va_arg (*args, vnet_policer_trace_t *);
  46
  47   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  48               t->sw_if_index, t->policer_index, t->next_index);
  49   return s;
  50 }
  51
  52 #define foreach_vnet_policer_error              \
  53 _(TRANSMIT, "Packets Transmitted")              \
  54 _(DROP, "Packets Dropped")
  55
  56 typedef enum
  57 {
  58 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  59   foreach_vnet_policer_error
  60 #undef _
  61     VNET_POLICER_N_ERROR,
  62 } vnet_policer_error_t;
  63
  64 static char *vnet_policer_error_strings[] = {
  65 #define _(sym,string) string,
  66   foreach_vnet_policer_error
  67 #undef _
  68 };
  69
  70 static_always_inline void
  71 vnet_policer_mark (vlib_buffer_t * b, u8 dscp)
  72 {
  73   ethernet_header_t *eh;
  74   ip4_header_t *ip4h;
  75   ip6_header_t *ip6h;
  76   u16 type;
  77
  78   eh = (ethernet_header_t *) b->data;
  79   type = clib_net_to_host_u16 (eh->type);
  80
  81   if (PREDICT_TRUE (type == ETHERNET_TYPE_IP4))
  82     {
  83       ip4h = (ip4_header_t *) & (b->data[sizeof (ethernet_header_t)]);;
  84       ip4h->tos &= IP4_NON_DSCP_BITS;
  85       ip4h->tos |= dscp << IP4_DSCP_SHIFT;
  86       ip4h->checksum = ip4_header_checksum (ip4h);
  87     }
  88   else
  89     {
  90       if (PREDICT_TRUE (type == ETHERNET_TYPE_IP6))
  91         {
  92           ip6h = (ip6_header_t *) & (b->data[sizeof (ethernet_header_t)]);
  93           ip6h->ip_version_traffic_class_and_flow_label &=
  94             clib_host_to_net_u32 (IP6_NON_DSCP_BITS);
  95           ip6h->ip_version_traffic_class_and_flow_label |=
  96             clib_host_to_net_u32 (dscp << IP6_DSCP_SHIFT);
  97         }
  98     }
  99 }
 100
 101 static_always_inline
 102   u8 vnet_policer_police (vlib_main_t * vm,
 103                           vlib_buffer_t * b,
 104                           u32 policer_index,
 105                           u64 time_in_policer_periods,
 106                           policer_result_e packet_color)
 107 {
 108   u8 act;
 109   u32 len;
 110   u32 col;
 111   policer_read_response_type_st *pol;
 112   vnet_policer_main_t *pm = &vnet_policer_main;
 113
 114   len = vlib_buffer_length_in_chain (vm, b);
 115   pol = &pm->policers[policer_index];
 116   col = vnet_police_packet (pol, len, packet_color, time_in_policer_periods);
 117   act = pol->action[col];
 118   if (PREDICT_TRUE (act == SSE2_QOS_ACTION_MARK_AND_TRANSMIT))
 119     vnet_policer_mark (b, pol->mark_dscp[col]);
 120
 121   return act;
 122 }
 123
 124 static inline uword
 125 vnet_policer_inline (vlib_main_t * vm,
 126                      vlib_node_runtime_t * node,
 127                      vlib_frame_t * frame, vnet_policer_index_t which)
 128 {
 129   u32 n_left_from, *from, *to_next;
 130   vnet_policer_next_t next_index;
 131   vnet_policer_main_t *pm = &vnet_policer_main;
 132   u64 time_in_policer_periods;
 133   u32 transmitted = 0;
 134
 135   time_in_policer_periods =
 136     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 137
 138   from = vlib_frame_vector_args (frame);
 139   n_left_from = frame->n_vectors;
 140   next_index = node->cached_next_index;
 141
 142   while (n_left_from > 0)
 143     {
 144       u32 n_left_to_next;
 145
 146       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 147
 148       while (n_left_from >= 4 && n_left_to_next >= 2)
 149         {
 150           u32 bi0, bi1;
 151           vlib_buffer_t *b0, *b1;
 152           u32 next0, next1;
 153           u32 sw_if_index0, sw_if_index1;
 154           u32 pi0 = 0, pi1 = 0;
 155           u8 act0, act1;
 156
 157           /* Prefetch next iteration. */
 158           {
 159             vlib_buffer_t *b2, *b3;
 160
 161             b2 = vlib_get_buffer (vm, from[2]);
 162             b3 = vlib_get_buffer (vm, from[3]);
 163
 164             vlib_prefetch_buffer_header (b2, LOAD);
 165             vlib_prefetch_buffer_header (b3, LOAD);
 166           }
 167
 168           /* speculatively enqueue b0 and b1 to the current next frame */
 169           to_next[0] = bi0 = from[0];
 170           to_next[1] = bi1 = from[1];
 171           from += 2;
 172           to_next += 2;
 173           n_left_from -= 2;
 174           n_left_to_next -= 2;
 175
 176           b0 = vlib_get_buffer (vm, bi0);
 177           b1 = vlib_get_buffer (vm, bi1);
 178
 179           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 180           next0 = VNET_POLICER_NEXT_TRANSMIT;
 181
 182           sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 183           next1 = VNET_POLICER_NEXT_TRANSMIT;
 184
 185
 186           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 187             {
 188               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 189               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 190             }
 191
 192           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 193             {
 194               pi0 = vnet_buffer (b0)->policer.index;
 195               pi1 = vnet_buffer (b1)->policer.index;
 196             }
 197
 198           if (which == VNET_POLICER_INDEX_BY_EITHER)
 199             {
 200               pi0 = vnet_buffer (b0)->policer.index;
 201               pi0 = (pi0 != ~0) ? pi0 :
 202                 pm->policer_index_by_sw_if_index[sw_if_index0];
 203               pi1 = vnet_buffer (b1)->policer.index;
 204               pi1 = (pi1 != ~0) ? pi1 :
 205                 pm->policer_index_by_sw_if_index[sw_if_index1];
 206             }
 207
 208           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 209                                       POLICE_CONFORM /* no chaining */ );
 210
 211           act1 = vnet_policer_police (vm, b1, pi1, time_in_policer_periods,
 212                                       POLICE_CONFORM /* no chaining */ );
 213
 214           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 215             {
 216               next0 = VNET_POLICER_NEXT_DROP;
 217               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 218             }
 219           else                  /* transmit or mark-and-transmit action */
 220             {
 221               transmitted++;
 222             }
 223
 224           if (PREDICT_FALSE (act1 == SSE2_QOS_ACTION_DROP))     /* drop action */
 225             {
 226               next1 = VNET_POLICER_NEXT_DROP;
 227               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 228             }
 229           else                  /* transmit or mark-and-transmit action */
 230             {
 231               transmitted++;
 232             }
 233
 234
 235           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)))
 236             {
 237               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 238                 {
 239                   vnet_policer_trace_t *t =
 240                     vlib_add_trace (vm, node, b0, sizeof (*t));
 241                   t->sw_if_index = sw_if_index0;
 242                   t->next_index = next0;
 243                 }
 244               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 245                 {
 246                   vnet_policer_trace_t *t =
 247                     vlib_add_trace (vm, node, b1, sizeof (*t));
 248                   t->sw_if_index = sw_if_index1;
 249                   t->next_index = next1;
 250                 }
 251             }
 252
 253           /* verify speculative enqueues, maybe switch current next frame */
 254           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 255                                            to_next, n_left_to_next,
 256                                            bi0, bi1, next0, next1);
 257         }
 258
 259       while (n_left_from > 0 && n_left_to_next > 0)
 260         {
 261           u32 bi0;
 262           vlib_buffer_t *b0;
 263           u32 next0;
 264           u32 sw_if_index0;
 265           u32 pi0 = 0;
 266           u8 act0;
 267
 268           bi0 = from[0];
 269           to_next[0] = bi0;
 270           from += 1;
 271           to_next += 1;
 272           n_left_from -= 1;
 273           n_left_to_next -= 1;
 274
 275           b0 = vlib_get_buffer (vm, bi0);
 276
 277           sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 278           next0 = VNET_POLICER_NEXT_TRANSMIT;
 279
 280           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 281             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 282
 283           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 284             pi0 = vnet_buffer (b0)->policer.index;
 285
 286           if (which == VNET_POLICER_INDEX_BY_EITHER)
 287             {
 288               pi0 = vnet_buffer (b0)->policer.index;
 289               pi0 = (pi0 != ~0) ? pi0 :
 290                 pm->policer_index_by_sw_if_index[sw_if_index0];
 291             }
 292
 293           act0 = vnet_policer_police (vm, b0, pi0, time_in_policer_periods,
 294                                       POLICE_CONFORM /* no chaining */ );
 295
 296           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))     /* drop action */
 297             {
 298               next0 = VNET_POLICER_NEXT_DROP;
 299               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 300             }
 301           else                  /* transmit or mark-and-transmit action */
 302             {
 303               transmitted++;
 304             }
 305
 306           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 307                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 308             {
 309               vnet_policer_trace_t *t =
 310                 vlib_add_trace (vm, node, b0, sizeof (*t));
 311               t->sw_if_index = sw_if_index0;
 312               t->next_index = next0;
 313               t->policer_index = pi0;
 314             }
 315
 316           /* verify speculative enqueue, maybe switch current next frame */
 317           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 318                                            to_next, n_left_to_next,
 319                                            bi0, next0);
 320         }
 321
 322       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 323     }
 324
 325   vlib_node_increment_counter (vm, node->node_index,
 326                                VNET_POLICER_ERROR_TRANSMIT, transmitted);
 327   return frame->n_vectors;
 328 }
 329
 330 uword
 331 vnet_policer_by_sw_if_index (vlib_main_t * vm,
 332                              vlib_node_runtime_t * node, vlib_frame_t * frame)
 333 {
 334   return vnet_policer_inline (vm, node, frame,
 335                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 336 }
 337
 338 uword
 339 vnet_policer_by_opaque (vlib_main_t * vm,
 340                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 341 {
 342   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_OPAQUE);
 343 }
 344
 345 uword
 346 vnet_policer_by_either (vlib_main_t * vm,
 347                         vlib_node_runtime_t * node, vlib_frame_t * frame)
 348 {
 349   return vnet_policer_inline (vm, node, frame, VNET_POLICER_INDEX_BY_EITHER);
 350 }
 351
 352 void
 353 vnet_policer_node_funcs_reference (void)
 354 {
 355 }
 356
 357
 358 #define TEST_CODE 1
 359
 360 #ifdef TEST_CODE
 361
 362 /* *INDENT-OFF* */
 363 VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = {
 364   .function = vnet_policer_by_sw_if_index,
 365   .name = "policer-by-sw-if-index",
 366   .vector_size = sizeof (u32),
 367   .format_trace = format_policer_trace,
 368   .type = VLIB_NODE_TYPE_INTERNAL,
 369
 370   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 371   .error_strings = vnet_policer_error_strings,
 372
 373   .n_next_nodes = VNET_POLICER_N_NEXT,
 374
 375   /* edit / add dispositions here */
 376   .next_nodes = {
 377     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 378     [VNET_POLICER_NEXT_DROP] = "error-drop",
 379   },
 380 };
 381
 382 VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
 383                               vnet_policer_by_sw_if_index);
 384 /* *INDENT-ON* */
 385
 386
 387 int
 388 test_policer_add_del (u32 rx_sw_if_index, u8 * config_name, int is_add)
 389 {
 390   vnet_policer_main_t *pm = &vnet_policer_main;
 391   policer_read_response_type_st *template;
 392   policer_read_response_type_st *policer;
 393   vnet_hw_interface_t *rxhi;
 394   uword *p;
 395
 396   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 397
 398   /* Make sure caller didn't pass a vlan subif, etc. */
 399   if (rxhi->sw_if_index != rx_sw_if_index)
 400     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 401
 402   if (is_add)
 403     {
 404
 405       p = hash_get_mem (pm->policer_config_by_name, config_name);
 406
 407       if (p == 0)
 408         return -2;
 409
 410       template = pool_elt_at_index (pm->policer_templates, p[0]);
 411
 412       vnet_hw_interface_rx_redirect_to_node
 413         (pm->vnet_main, rxhi->hw_if_index, policer_by_sw_if_index_node.index);
 414
 415       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 416
 417       policer[0] = template[0];
 418
 419       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 420       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 421         = policer - pm->policers;
 422     }
 423   else
 424     {
 425       u32 pi;
 426       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 427                                              rxhi->hw_if_index,
 428                                              ~0 /* disable */ );
 429
 430       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 431       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 432       pool_put_index (pm->policers, pi);
 433     }
 434
 435   return 0;
 436 }
 437
 438 static clib_error_t *
 439 test_policer_command_fn (vlib_main_t * vm,
 440                          unformat_input_t * input, vlib_cli_command_t * cmd)
 441 {
 442   vnet_policer_main_t *pm = &vnet_policer_main;
 443   unformat_input_t _line_input, *line_input = &_line_input;
 444   u32 rx_sw_if_index;
 445   int rv;
 446   u8 *config_name = 0;
 447   int rx_set = 0;
 448   int is_add = 1;
 449   int is_show = 0;
 450
 451   /* Get a line of input. */
 452   if (!unformat_user (input, unformat_line_input, line_input))
 453     return 0;
 454
 455   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 456     {
 457       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 458                     pm->vnet_main, &rx_sw_if_index))
 459         rx_set = 1;
 460       else if (unformat (line_input, "show"))
 461         is_show = 1;
 462       else if (unformat (line_input, "policer %s", &config_name))
 463         ;
 464       else if (unformat (line_input, "del"))
 465         is_add = 0;
 466       else
 467         break;
 468     }
 469
 470   if (rx_set == 0)
 471     return clib_error_return (0, "interface not set");
 472
 473   if (is_show)
 474     {
 475       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 476       policer_read_response_type_st *policer;
 477       policer = pool_elt_at_index (pm->policers, pi);
 478
 479       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 480       return 0;
 481     }
 482
 483   if (is_add && config_name == 0)
 484     {
 485       return clib_error_return (0, "policer config name required");
 486     }
 487
 488   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 489
 490   switch (rv)
 491     {
 492     case 0:
 493       break;
 494
 495     default:
 496       return clib_error_return
 497         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 498     }
 499
 500   return 0;
 501 }
 502
 503 /* *INDENT-OFF* */
 504 VLIB_CLI_COMMAND (test_patch_command, static) = {
 505     .path = "test policer",
 506     .short_help =
 507     "intfc <intfc> policer <policer-config-name> [del]",
 508     .function = test_policer_command_fn,
 509 };
 510 /* *INDENT-ON* */
 511
 512 #endif /* TEST_CODE */
 513
 514
 515 typedef struct
 516 {
 517   u32 sw_if_index;
 518   u32 next_index;
 519   u32 table_index;
 520   u32 offset;
 521   u32 policer_index;
 522 } policer_classify_trace_t;
 523
 524 static u8 *
 525 format_policer_classify_trace (u8 * s, va_list * args)
 526 {
 527   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 528   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 529   policer_classify_trace_t *t = va_arg (*args, policer_classify_trace_t *);
 530
 531   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 532               " policer_index %d",
 533               t->sw_if_index, t->next_index, t->table_index, t->offset,
 534               t->policer_index);
 535   return s;
 536 }
 537
 538 #define foreach_policer_classify_error                 \
 539 _(MISS, "Policer classify misses")                     \
 540 _(HIT, "Policer classify hits")                        \
 541 _(CHAIN_HIT, "Polcier classify hits after chain walk") \
 542 _(DROP, "Policer classify action drop")
 543
 544 typedef enum
 545 {
 546 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 547   foreach_policer_classify_error
 548 #undef _
 549     POLICER_CLASSIFY_N_ERROR,
 550 } policer_classify_error_t;
 551
 552 static char *policer_classify_error_strings[] = {
 553 #define _(sym,string) string,
 554   foreach_policer_classify_error
 555 #undef _
 556 };
 557
 558 static inline uword
 559 policer_classify_inline (vlib_main_t * vm,
 560                          vlib_node_runtime_t * node,
 561                          vlib_frame_t * frame,
 562                          policer_classify_table_id_t tid)
 563 {
 564   u32 n_left_from, *from, *to_next;
 565   policer_classify_next_index_t next_index;
 566   policer_classify_main_t *pcm = &policer_classify_main;
 567   vnet_classify_main_t *vcm = pcm->vnet_classify_main;
 568   f64 now = vlib_time_now (vm);
 569   u32 hits = 0;
 570   u32 misses = 0;
 571   u32 chain_hits = 0;
 572   u32 drop = 0;
 573   u32 n_next_nodes;
 574   u64 time_in_policer_periods;
 575
 576   time_in_policer_periods =
 577     clib_cpu_time_now () >> POLICER_TICKS_PER_PERIOD_SHIFT;
 578
 579   n_next_nodes = node->n_next_nodes;
 580
 581   from = vlib_frame_vector_args (frame);
 582   n_left_from = frame->n_vectors;
 583
 584   /* First pass: compute hashes */
 585   while (n_left_from > 2)
 586     {
 587       vlib_buffer_t *b0, *b1;
 588       u32 bi0, bi1;
 589       u8 *h0, *h1;
 590       u32 sw_if_index0, sw_if_index1;
 591       u32 table_index0, table_index1;
 592       vnet_classify_table_t *t0, *t1;
 593
 594       /* Prefetch next iteration */
 595       {
 596         vlib_buffer_t *p1, *p2;
 597
 598         p1 = vlib_get_buffer (vm, from[1]);
 599         p2 = vlib_get_buffer (vm, from[2]);
 600
 601         vlib_prefetch_buffer_header (p1, STORE);
 602         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 603         vlib_prefetch_buffer_header (p2, STORE);
 604         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 605       }
 606
 607       bi0 = from[0];
 608       b0 = vlib_get_buffer (vm, bi0);
 609       h0 = b0->data;
 610
 611       bi1 = from[1];
 612       b1 = vlib_get_buffer (vm, bi1);
 613       h1 = b1->data;
 614
 615       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 616       table_index0 =
 617         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 618
 619       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 620       table_index1 =
 621         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 622
 623       t0 = pool_elt_at_index (vcm->tables, table_index0);
 624
 625       t1 = pool_elt_at_index (vcm->tables, table_index1);
 626
 627       vnet_buffer (b0)->l2_classify.hash =
 628         vnet_classify_hash_packet (t0, (u8 *) h0);
 629
 630       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 631
 632       vnet_buffer (b1)->l2_classify.hash =
 633         vnet_classify_hash_packet (t1, (u8 *) h1);
 634
 635       vnet_classify_prefetch_bucket (t1, vnet_buffer (b1)->l2_classify.hash);
 636
 637       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 638
 639       vnet_buffer (b1)->l2_classify.table_index = table_index1;
 640
 641       from += 2;
 642       n_left_from -= 2;
 643     }
 644
 645   while (n_left_from > 0)
 646     {
 647       vlib_buffer_t *b0;
 648       u32 bi0;
 649       u8 *h0;
 650       u32 sw_if_index0;
 651       u32 table_index0;
 652       vnet_classify_table_t *t0;
 653
 654       bi0 = from[0];
 655       b0 = vlib_get_buffer (vm, bi0);
 656       h0 = b0->data;
 657
 658       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 659       table_index0 =
 660         pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 661
 662       t0 = pool_elt_at_index (vcm->tables, table_index0);
 663       vnet_buffer (b0)->l2_classify.hash =
 664         vnet_classify_hash_packet (t0, (u8 *) h0);
 665
 666       vnet_buffer (b0)->l2_classify.table_index = table_index0;
 667       vnet_classify_prefetch_bucket (t0, vnet_buffer (b0)->l2_classify.hash);
 668
 669       from++;
 670       n_left_from--;
 671     }
 672
 673   next_index = node->cached_next_index;
 674   from = vlib_frame_vector_args (frame);
 675   n_left_from = frame->n_vectors;
 676
 677   while (n_left_from > 0)
 678     {
 679       u32 n_left_to_next;
 680
 681       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 682
 683       /* Not enough load/store slots to dual loop... */
 684       while (n_left_from > 0 && n_left_to_next > 0)
 685         {
 686           u32 bi0;
 687           vlib_buffer_t *b0;
 688           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 689           u32 table_index0;
 690           vnet_classify_table_t *t0;
 691           vnet_classify_entry_t *e0;
 692           u64 hash0;
 693           u8 *h0;
 694           u8 act0;
 695
 696           /* Stride 3 seems to work best */
 697           if (PREDICT_TRUE (n_left_from > 3))
 698             {
 699               vlib_buffer_t *p1 = vlib_get_buffer (vm, from[3]);
 700               vnet_classify_table_t *tp1;
 701               u32 table_index1;
 702               u64 phash1;
 703
 704               table_index1 = vnet_buffer (p1)->l2_classify.table_index;
 705
 706               if (PREDICT_TRUE (table_index1 != ~0))
 707                 {
 708                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 709                   phash1 = vnet_buffer (p1)->l2_classify.hash;
 710                   vnet_classify_prefetch_entry (tp1, phash1);
 711                 }
 712             }
 713
 714           /* Speculatively enqueue b0 to the current next frame */
 715           bi0 = from[0];
 716           to_next[0] = bi0;
 717           from += 1;
 718           to_next += 1;
 719           n_left_from -= 1;
 720           n_left_to_next -= 1;
 721
 722           b0 = vlib_get_buffer (vm, bi0);
 723           h0 = b0->data;
 724           table_index0 = vnet_buffer (b0)->l2_classify.table_index;
 725           e0 = 0;
 726           t0 = 0;
 727
 728           if (tid == POLICER_CLASSIFY_TABLE_L2)
 729             {
 730               /* Feature bitmap update */
 731               vnet_buffer (b0)->l2.feature_bitmap &=
 732                 ~L2INPUT_FEAT_POLICER_CLAS;
 733               /* Determine the next node */
 734               next0 =
 735                 feat_bitmap_get_next_node_index (pcm->feat_next_node_index,
 736                                                  vnet_buffer (b0)->
 737                                                  l2.feature_bitmap);
 738             }
 739           else
 740             vnet_get_config_data (pcm->vnet_config_main[tid],
 741                                   &b0->current_config_index, &next0,
 742                                   /* # bytes of config data */ 0);
 743
 744           vnet_buffer (b0)->l2_classify.opaque_index = ~0;
 745
 746           if (PREDICT_TRUE (table_index0 != ~0))
 747             {
 748               hash0 = vnet_buffer (b0)->l2_classify.hash;
 749               t0 = pool_elt_at_index (vcm->tables, table_index0);
 750               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 751
 752               if (e0)
 753                 {
 754                   act0 = vnet_policer_police (vm,
 755                                               b0,
 756                                               e0->next_index,
 757                                               time_in_policer_periods,
 758                                               e0->opaque_index);
 759                   if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 760                     {
 761                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 762                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 763                       drop++;
 764                     }
 765                   hits++;
 766                 }
 767               else
 768                 {
 769                   while (1)
 770                     {
 771                       if (PREDICT_TRUE (t0->next_table_index != ~0))
 772                         {
 773                           t0 = pool_elt_at_index (vcm->tables,
 774                                                   t0->next_table_index);
 775                         }
 776                       else
 777                         {
 778                           next0 = (t0->miss_next_index < n_next_nodes) ?
 779                             t0->miss_next_index : next0;
 780                           misses++;
 781                           break;
 782                         }
 783
 784                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 785                       e0 =
 786                         vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 787                       if (e0)
 788                         {
 789                           act0 = vnet_policer_police (vm,
 790                                                       b0,
 791                                                       e0->next_index,
 792                                                       time_in_policer_periods,
 793                                                       e0->opaque_index);
 794                           if (PREDICT_FALSE (act0 == SSE2_QOS_ACTION_DROP))
 795                             {
 796                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 797                               b0->error =
 798                                 node->errors[POLICER_CLASSIFY_ERROR_DROP];
 799                               drop++;
 800                             }
 801                           hits++;
 802                           chain_hits++;
 803                           break;
 804                         }
 805                     }
 806                 }
 807             }
 808           if (PREDICT_FALSE ((node->flags & VLIB_NODE_FLAG_TRACE)
 809                              && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 810             {
 811               policer_classify_trace_t *t =
 812                 vlib_add_trace (vm, node, b0, sizeof (*t));
 813               t->sw_if_index = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 814               t->next_index = next0;
 815               t->table_index = t0 ? t0 - vcm->tables : ~0;
 816               t->offset = (e0 && t0) ? vnet_classify_get_offset (t0, e0) : ~0;
 817               t->policer_index = e0 ? e0->next_index : ~0;
 818             }
 819
 820           /* Verify speculative enqueue, maybe switch current next frame */
 821           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 822                                            n_left_to_next, bi0, next0);
 823         }
 824
 825       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 826     }
 827
 828   vlib_node_increment_counter (vm, node->node_index,
 829                                POLICER_CLASSIFY_ERROR_MISS, misses);
 830   vlib_node_increment_counter (vm, node->node_index,
 831                                POLICER_CLASSIFY_ERROR_HIT, hits);
 832   vlib_node_increment_counter (vm, node->node_index,
 833                                POLICER_CLASSIFY_ERROR_CHAIN_HIT, chain_hits);
 834   vlib_node_increment_counter (vm, node->node_index,
 835                                POLICER_CLASSIFY_ERROR_DROP, drop);
 836
 837   return frame->n_vectors;
 838 }
 839
 840 static uword
 841 ip4_policer_classify (vlib_main_t * vm,
 842                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 843 {
 844   return policer_classify_inline (vm, node, frame,
 845                                   POLICER_CLASSIFY_TABLE_IP4);
 846 }
 847
 848 /* *INDENT-OFF* */
 849 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 850   .function = ip4_policer_classify,
 851   .name = "ip4-policer-classify",
 852   .vector_size = sizeof (u32),
 853   .format_trace = format_policer_classify_trace,
 854   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 855   .error_strings = policer_classify_error_strings,
 856   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 857   .next_nodes = {
 858     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 859   },
 860 };
 861
 862 VLIB_NODE_FUNCTION_MULTIARCH (ip4_policer_classify_node, ip4_policer_classify);
 863 /* *INDENT-ON* */
 864
 865 static uword
 866 ip6_policer_classify (vlib_main_t * vm,
 867                       vlib_node_runtime_t * node, vlib_frame_t * frame)
 868 {
 869   return policer_classify_inline (vm, node, frame,
 870                                   POLICER_CLASSIFY_TABLE_IP6);
 871 }
 872
 873 /* *INDENT-OFF* */
 874 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 875   .function = ip6_policer_classify,
 876   .name = "ip6-policer-classify",
 877   .vector_size = sizeof (u32),
 878   .format_trace = format_policer_classify_trace,
 879   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 880   .error_strings = policer_classify_error_strings,
 881   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 882   .next_nodes = {
 883     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 884   },
 885 };
 886
 887 VLIB_NODE_FUNCTION_MULTIARCH (ip6_policer_classify_node, ip6_policer_classify);
 888 /* *INDENT-ON* */
 889
 890 static uword
 891 l2_policer_classify (vlib_main_t * vm,
 892                      vlib_node_runtime_t * node, vlib_frame_t * frame)
 893 {
 894   return policer_classify_inline (vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 895 }
 896
 897 VLIB_REGISTER_NODE (l2_policer_classify_node) =
 898 {
 899   .function = l2_policer_classify,.name = "l2-policer-classify",.vector_size =
 900     sizeof (u32),.format_trace = format_policer_classify_trace,.n_errors =
 901     ARRAY_LEN (policer_classify_error_strings),.error_strings =
 902     policer_classify_error_strings,.n_next_nodes =
 903     POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,.next_nodes =
 904   {
 905   [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",}
 906 ,};
 907
 908 VLIB_NODE_FUNCTION_MULTIARCH (l2_policer_classify_node, l2_policer_classify);
 909
 910
 911 static clib_error_t *
 912 policer_classify_init (vlib_main_t * vm)
 913 {
 914   policer_classify_main_t *pcm = &policer_classify_main;
 915
 916   pcm->vlib_main = vm;
 917   pcm->vnet_main = vnet_get_main ();
 918   pcm->vnet_classify_main = &vnet_classify_main;
 919
 920   /* Initialize L2 feature next-node indexes */
 921   feat_bitmap_init_next_nodes (vm,
 922                                l2_policer_classify_node.index,
 923                                L2INPUT_N_FEAT,
 924                                l2input_get_feat_names (),
 925                                pcm->feat_next_node_index);
 926
 927   return 0;
 928 }
 929
 930 VLIB_INIT_FUNCTION (policer_classify_init);
 931
 932 /*
 933  * fd.io coding-style-patch-verification: ON
 934  *
 935  * Local Variables:
 936  * eval: (c-set-style "gnu")
 937  * End:
 938  */