vnet/vnet/policer/node_funcs.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15
  16 #include <stdint.h>
  17
  18 #include <vlib/vlib.h>
  19 #include <vnet/vnet.h>
  20 #include <vnet/policer/policer.h>
  21 #include <vnet/ip/ip.h>
  22 #include <vnet/classify/policer_classify.h>
  23 #include <vnet/classify/vnet_classify.h>
  24
  25 #define IP4_NON_DSCP_BITS 0x03
  26 #define IP4_DSCP_SHIFT    2
  27 #define IP6_NON_DSCP_BITS 0xf03fffff
  28 #define IP6_DSCP_SHIFT    22
  29
  30 /* Dispatch functions meant to be instantiated elsewhere */
  31
  32 typedef struct {
  33   u32 next_index;
  34   u32 sw_if_index;
  35   u32 policer_index;
  36 } vnet_policer_trace_t;
  37
  38 /* packet trace format function */
  39 static u8 * format_policer_trace (u8 * s, va_list * args)
  40 {
  41   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  42   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  43   vnet_policer_trace_t * t = va_arg (*args, vnet_policer_trace_t *);
  44
  45   s = format (s, "VNET_POLICER: sw_if_index %d policer_index %d next %d",
  46               t->sw_if_index, t->policer_index, t->next_index);
  47   return s;
  48 }
  49
  50 #define foreach_vnet_policer_error              \
  51 _(TRANSMIT, "Packets Transmitted")              \
  52 _(DROP, "Packets Dropped")
  53
  54 typedef enum {
  55 #define _(sym,str) VNET_POLICER_ERROR_##sym,
  56   foreach_vnet_policer_error
  57 #undef _
  58   VNET_POLICER_N_ERROR,
  59 } vnet_policer_error_t;
  60
  61 static char * vnet_policer_error_strings[] = {
  62 #define _(sym,string) string,
  63   foreach_vnet_policer_error
  64 #undef _
  65 };
  66
  67 static_always_inline
  68 void vnet_policer_mark (vlib_buffer_t * b, u8 dscp)
  69 {
  70   ethernet_header_t * eh;
  71   ip4_header_t * ip4h;
  72   ip6_header_t * ip6h;
  73   u16 type;
  74
  75   eh = (ethernet_header_t *) b->data;
  76   type = clib_net_to_host_u16 (eh->type);
  77
  78   if (PREDICT_TRUE(type == ETHERNET_TYPE_IP4))
  79     {
  80       ip4h = (ip4_header_t *) &(b->data[sizeof(ethernet_header_t)]);;
  81       ip4h->tos &= IP4_NON_DSCP_BITS;
  82       ip4h->tos |= dscp << IP4_DSCP_SHIFT;
  83       ip4h->checksum = ip4_header_checksum (ip4h);
  84     }
  85   else
  86     {
  87       if (PREDICT_TRUE(type == ETHERNET_TYPE_IP6))
  88         {
  89           ip6h = (ip6_header_t *) &(b->data[sizeof(ethernet_header_t)]);
  90           ip6h->ip_version_traffic_class_and_flow_label &=
  91             clib_host_to_net_u32(IP6_NON_DSCP_BITS);
  92           ip6h->ip_version_traffic_class_and_flow_label |=
  93             clib_host_to_net_u32(dscp << IP6_DSCP_SHIFT);
  94         }
  95     }
  96 }
  97
  98 static_always_inline
  99 u8 vnet_policer_police (vlib_main_t * vm,
 100                         vlib_buffer_t * b,
 101                         u32 policer_index,
 102                         u64 time_in_policer_periods,
 103                         policer_result_e packet_color)
 104 {
 105   u8 act;
 106   u32 len;
 107   u32 col;
 108   policer_read_response_type_st *pol;
 109   vnet_policer_main_t * pm = &vnet_policer_main;
 110
 111   len = vlib_buffer_length_in_chain (vm, b);
 112   pol = &pm->policers [policer_index];
 113   col = vnet_police_packet (pol, len,
 114                             packet_color,
 115                             time_in_policer_periods);
 116   act = pol->action[col];
 117   if (PREDICT_TRUE(act == SSE2_QOS_ACTION_MARK_AND_TRANSMIT))
 118       vnet_policer_mark(b, pol->mark_dscp[col]);
 119
 120   return act;
 121  }
 122
 123 static inline
 124 uword vnet_policer_inline (vlib_main_t * vm,
 125                            vlib_node_runtime_t * node,
 126                            vlib_frame_t * frame,
 127                            vnet_policer_index_t which)
 128 {
 129   u32 n_left_from, * from, * to_next;
 130   vnet_policer_next_t next_index;
 131   vnet_policer_main_t * pm = &vnet_policer_main;
 132   u64 time_in_policer_periods;
 133   u32 transmitted = 0;
 134
 135   time_in_policer_periods =
 136     clib_cpu_time_now() >> POLICER_TICKS_PER_PERIOD_SHIFT;
 137
 138   from = vlib_frame_vector_args (frame);
 139   n_left_from = frame->n_vectors;
 140   next_index = node->cached_next_index;
 141
 142   while (n_left_from > 0)
 143     {
 144       u32 n_left_to_next;
 145
 146       vlib_get_next_frame (vm, node, next_index,
 147                            to_next, n_left_to_next);
 148
 149       while (n_left_from >= 4 && n_left_to_next >= 2)
 150         {
 151           u32 bi0, bi1;
 152           vlib_buffer_t * b0, * b1;
 153           u32 next0, next1;
 154           u32 sw_if_index0, sw_if_index1;
 155           u32 pi0 = 0, pi1 = 0;
 156           u8 act0, act1;
 157
 158           /* Prefetch next iteration. */
 159           {
 160             vlib_buffer_t * b2, * b3;
 161
 162             b2 = vlib_get_buffer (vm, from[2]);
 163             b3 = vlib_get_buffer (vm, from[3]);
 164
 165             vlib_prefetch_buffer_header (b2, LOAD);
 166             vlib_prefetch_buffer_header (b3, LOAD);
 167           }
 168
 169           /* speculatively enqueue b0 and b1 to the current next frame */
 170           to_next[0] = bi0 = from[0];
 171           to_next[1] = bi1 = from[1];
 172           from += 2;
 173           to_next += 2;
 174           n_left_from -= 2;
 175           n_left_to_next -= 2;
 176
 177           b0 = vlib_get_buffer (vm, bi0);
 178           b1 = vlib_get_buffer (vm, bi1);
 179
 180           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 181           next0 = VNET_POLICER_NEXT_TRANSMIT;
 182
 183           sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 184           next1 = VNET_POLICER_NEXT_TRANSMIT;
 185
 186
 187           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 188             {
 189               pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 190               pi1 = pm->policer_index_by_sw_if_index[sw_if_index1];
 191             }
 192
 193           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 194             {
 195               pi0 = vnet_buffer(b0)->policer.index;
 196               pi1 = vnet_buffer(b1)->policer.index;
 197             }
 198
 199           if (which == VNET_POLICER_INDEX_BY_EITHER)
 200             {
 201               pi0 = vnet_buffer(b0)->policer.index;
 202               pi0 = (pi0 != ~0) ? pi0 :
 203                 pm->policer_index_by_sw_if_index [sw_if_index0];
 204               pi1 = vnet_buffer(b1)->policer.index;
 205               pi1 = (pi1 != ~0) ? pi1 :
 206                 pm->policer_index_by_sw_if_index [sw_if_index1];
 207             }
 208
 209           act0 = vnet_policer_police(vm, b0, pi0, time_in_policer_periods,
 210             POLICE_CONFORM /* no chaining */);
 211
 212           act1 = vnet_policer_police(vm, b1, pi1, time_in_policer_periods,
 213             POLICE_CONFORM /* no chaining */);
 214
 215           if (PREDICT_FALSE(act0 == SSE2_QOS_ACTION_DROP)) /* drop action */
 216             {
 217               next0 = VNET_POLICER_NEXT_DROP;
 218               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 219             }
 220           else /* transmit or mark-and-transmit action */
 221             {
 222               transmitted++;
 223             }
 224
 225           if (PREDICT_FALSE(act1 == SSE2_QOS_ACTION_DROP)) /* drop action */
 226             {
 227               next1 = VNET_POLICER_NEXT_DROP;
 228               b1->error = node->errors[VNET_POLICER_ERROR_DROP];
 229             }
 230           else /* transmit or mark-and-transmit action */
 231             {
 232               transmitted++;
 233             }
 234
 235
 236           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)))
 237             {
 238               if (b0->flags & VLIB_BUFFER_IS_TRACED)
 239                 {
 240                   vnet_policer_trace_t *t =
 241                     vlib_add_trace (vm, node, b0, sizeof (*t));
 242                   t->sw_if_index = sw_if_index0;
 243                   t->next_index = next0;
 244                 }
 245               if (b1->flags & VLIB_BUFFER_IS_TRACED)
 246                 {
 247                   vnet_policer_trace_t *t =
 248                     vlib_add_trace (vm, node, b1, sizeof (*t));
 249                   t->sw_if_index = sw_if_index1;
 250                   t->next_index = next1;
 251                 }
 252             }
 253
 254           /* verify speculative enqueues, maybe switch current next frame */
 255           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 256                                            to_next, n_left_to_next,
 257                                            bi0, bi1, next0, next1);
 258         }
 259
 260       while (n_left_from > 0 && n_left_to_next > 0)
 261         {
 262           u32 bi0;
 263           vlib_buffer_t * b0;
 264           u32 next0;
 265           u32 sw_if_index0;
 266           u32 pi0 = 0;
 267           u8 act0;
 268
 269           bi0 = from[0];
 270           to_next[0] = bi0;
 271           from += 1;
 272           to_next += 1;
 273           n_left_from -= 1;
 274           n_left_to_next -= 1;
 275
 276           b0 = vlib_get_buffer (vm, bi0);
 277
 278           sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 279           next0 = VNET_POLICER_NEXT_TRANSMIT;
 280
 281           if (which == VNET_POLICER_INDEX_BY_SW_IF_INDEX)
 282             pi0 = pm->policer_index_by_sw_if_index[sw_if_index0];
 283
 284           if (which == VNET_POLICER_INDEX_BY_OPAQUE)
 285             pi0 = vnet_buffer(b0)->policer.index;
 286
 287           if (which == VNET_POLICER_INDEX_BY_EITHER)
 288             {
 289               pi0 = vnet_buffer(b0)->policer.index;
 290               pi0 = (pi0 != ~0) ? pi0 :
 291                 pm->policer_index_by_sw_if_index [sw_if_index0];
 292             }
 293
 294           act0 = vnet_policer_police(vm, b0, pi0, time_in_policer_periods,
 295             POLICE_CONFORM /* no chaining */);
 296
 297           if (PREDICT_FALSE(act0 == SSE2_QOS_ACTION_DROP)) /* drop action */
 298             {
 299               next0 = VNET_POLICER_NEXT_DROP;
 300               b0->error = node->errors[VNET_POLICER_ERROR_DROP];
 301             }
 302           else /* transmit or mark-and-transmit action */
 303             {
 304               transmitted++;
 305             }
 306
 307           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 308                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 309             {
 310               vnet_policer_trace_t *t =
 311                 vlib_add_trace (vm, node, b0, sizeof (*t));
 312               t->sw_if_index = sw_if_index0;
 313               t->next_index = next0;
 314               t->policer_index = pi0;
 315             }
 316
 317           /* verify speculative enqueue, maybe switch current next frame */
 318           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 319                                            to_next, n_left_to_next,
 320                                            bi0, next0);
 321         }
 322
 323       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 324     }
 325
 326   vlib_node_increment_counter (vm, node->node_index,
 327                                VNET_POLICER_ERROR_TRANSMIT,
 328                                transmitted);
 329   return frame->n_vectors;
 330 }
 331
 332 uword vnet_policer_by_sw_if_index (vlib_main_t * vm,
 333                                    vlib_node_runtime_t * node,
 334                                    vlib_frame_t * frame)
 335 {
 336   return vnet_policer_inline (vm, node, frame,
 337                               VNET_POLICER_INDEX_BY_SW_IF_INDEX);
 338 }
 339
 340 uword vnet_policer_by_opaque (vlib_main_t * vm,
 341                               vlib_node_runtime_t * node,
 342                               vlib_frame_t * frame)
 343 {
 344   return vnet_policer_inline (vm, node, frame,
 345                               VNET_POLICER_INDEX_BY_OPAQUE);
 346 }
 347
 348 uword vnet_policer_by_either (vlib_main_t * vm,
 349                               vlib_node_runtime_t * node,
 350                               vlib_frame_t * frame)
 351 {
 352   return vnet_policer_inline (vm, node, frame,
 353                               VNET_POLICER_INDEX_BY_EITHER);
 354 }
 355
 356 void vnet_policer_node_funcs_reference (void) { }
 357
 358
 359 #define TEST_CODE 1
 360
 361 #ifdef TEST_CODE
 362
 363 VLIB_REGISTER_NODE (policer_by_sw_if_index_node, static) = {
 364   .function = vnet_policer_by_sw_if_index,
 365   .name = "policer-by-sw-if-index",
 366   .vector_size = sizeof (u32),
 367   .format_trace = format_policer_trace,
 368   .type = VLIB_NODE_TYPE_INTERNAL,
 369
 370   .n_errors = ARRAY_LEN(vnet_policer_error_strings),
 371   .error_strings = vnet_policer_error_strings,
 372
 373   .n_next_nodes = VNET_POLICER_N_NEXT,
 374
 375   /* edit / add dispositions here */
 376   .next_nodes = {
 377     [VNET_POLICER_NEXT_TRANSMIT] = "ethernet-input",
 378     [VNET_POLICER_NEXT_DROP] = "error-drop",
 379   },
 380 };
 381
 382 VLIB_NODE_FUNCTION_MULTIARCH (policer_by_sw_if_index_node,
 383                               vnet_policer_by_sw_if_index);
 384
 385
 386 int test_policer_add_del (u32 rx_sw_if_index, u8 *config_name,
 387                           int is_add)
 388 {
 389   vnet_policer_main_t * pm = &vnet_policer_main;
 390   policer_read_response_type_st * template;
 391   policer_read_response_type_st * policer;
 392   vnet_hw_interface_t * rxhi;
 393   uword * p;
 394
 395   rxhi = vnet_get_sup_hw_interface (pm->vnet_main, rx_sw_if_index);
 396
 397   /* Make sure caller didn't pass a vlan subif, etc. */
 398   if (rxhi->sw_if_index != rx_sw_if_index)
 399     return VNET_API_ERROR_INVALID_SW_IF_INDEX;
 400
 401   if (is_add)
 402     {
 403
 404       p = hash_get_mem (pm->policer_config_by_name, config_name);
 405
 406       if (p == 0)
 407         return -2;
 408
 409       template = pool_elt_at_index (pm->policer_templates, p[0]);
 410
 411       vnet_hw_interface_rx_redirect_to_node
 412         (pm->vnet_main,
 413          rxhi->hw_if_index,
 414          policer_by_sw_if_index_node.index);
 415
 416       pool_get_aligned (pm->policers, policer, CLIB_CACHE_LINE_BYTES);
 417
 418       policer[0] = template[0];
 419
 420       vec_validate (pm->policer_index_by_sw_if_index, rx_sw_if_index);
 421       pm->policer_index_by_sw_if_index[rx_sw_if_index]
 422           = policer - pm->policers;
 423     }
 424   else
 425     {
 426       u32 pi;
 427       vnet_hw_interface_rx_redirect_to_node (pm->vnet_main,
 428                                              rxhi->hw_if_index,
 429                                              ~0 /* disable */);
 430
 431       pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 432       pm->policer_index_by_sw_if_index[rx_sw_if_index] = ~0;
 433       pool_put_index (pm->policers, pi);
 434     }
 435
 436   return 0;
 437 }
 438
 439 static clib_error_t *
 440 test_policer_command_fn (vlib_main_t * vm,
 441                          unformat_input_t * input,
 442                          vlib_cli_command_t * cmd)
 443 {
 444   vnet_policer_main_t * pm = &vnet_policer_main;
 445   unformat_input_t _line_input, * line_input = &_line_input;
 446   u32 rx_sw_if_index;
 447   int rv;
 448   u8 * config_name = 0;
 449   int rx_set = 0;
 450   int is_add = 1;
 451   int is_show = 0;
 452
 453   /* Get a line of input. */
 454   if (! unformat_user (input, unformat_line_input, line_input))
 455     return 0;
 456
 457   while (unformat_check_input (line_input) != UNFORMAT_END_OF_INPUT)
 458     {
 459       if (unformat (line_input, "intfc %U", unformat_vnet_sw_interface,
 460                     pm->vnet_main, &rx_sw_if_index))
 461         rx_set = 1;
 462       else if (unformat (line_input, "show"))
 463         is_show=1;
 464       else if (unformat (line_input, "policer %s", &config_name))
 465         ;
 466       else if (unformat (line_input, "del"))
 467         is_add = 0;
 468       else break;
 469     }
 470
 471   if (rx_set == 0)
 472     return clib_error_return (0, "interface not set");
 473
 474   if (is_show)
 475     {
 476       u32 pi = pm->policer_index_by_sw_if_index[rx_sw_if_index];
 477       policer_read_response_type_st * policer;
 478       policer = pool_elt_at_index (pm->policers, pi);
 479
 480       vlib_cli_output (vm, "%U", format_policer_instance, policer);
 481       return 0;
 482     }
 483
 484   if (is_add && config_name == 0)
 485     {
 486       return clib_error_return (0, "policer config name required");
 487     }
 488
 489   rv = test_policer_add_del (rx_sw_if_index, config_name, is_add);
 490
 491   switch (rv)
 492     {
 493     case 0:
 494       break;
 495
 496     default:
 497       return clib_error_return
 498         (0, "WARNING: vnet_vnet_policer_add_del returned %d", rv);
 499     }
 500
 501   return 0;
 502 }
 503
 504 VLIB_CLI_COMMAND (test_patch_command, static) = {
 505     .path = "test policer",
 506     .short_help =
 507     "intfc <intfc> policer <policer-config-name> [del]",
 508     .function = test_policer_command_fn,
 509 };
 510
 511
 512 #endif /* TEST_CODE */
 513
 514
 515 typedef struct {
 516   u32 sw_if_index;
 517   u32 next_index;
 518   u32 table_index;
 519   u32 offset;
 520   u32 policer_index;
 521 } policer_classify_trace_t;
 522
 523 static u8 *
 524 format_policer_classify_trace (u8 * s, va_list * args)
 525 {
 526   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
 527   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
 528   policer_classify_trace_t * t = va_arg (*args, policer_classify_trace_t *);
 529
 530   s = format (s, "POLICER_CLASSIFY: sw_if_index %d next %d table %d offset %d"
 531               " policer_index %d",
 532               t->sw_if_index, t->next_index, t->table_index, t->offset,
 533               t->policer_index);
 534   return s;
 535 }
 536
 537 #define foreach_policer_classify_error                 \
 538 _(MISS, "Policer classify misses")                     \
 539 _(HIT, "Policer classify hits")                        \
 540 _(CHAIN_HIT, "Polcier classify hits after chain walk") \
 541 _(DROP, "Policer classify action drop")
 542
 543 typedef enum {
 544 #define _(sym,str) POLICER_CLASSIFY_ERROR_##sym,
 545   foreach_policer_classify_error
 546 #undef _
 547   POLICER_CLASSIFY_N_ERROR,
 548 } policer_classify_error_t;
 549
 550 static char * policer_classify_error_strings[] = {
 551 #define _(sym,string) string,
 552   foreach_policer_classify_error
 553 #undef _
 554 };
 555
 556 static inline uword
 557 policer_classify_inline (vlib_main_t * vm,
 558                          vlib_node_runtime_t * node,
 559                          vlib_frame_t * frame,
 560                          policer_classify_table_id_t tid)
 561 {
 562   u32 n_left_from, * from, * to_next;
 563   policer_classify_next_index_t next_index;
 564   policer_classify_main_t * pcm = &policer_classify_main;
 565   vnet_classify_main_t * vcm = pcm->vnet_classify_main;
 566   f64 now = vlib_time_now (vm);
 567   u32 hits = 0;
 568   u32 misses = 0;
 569   u32 chain_hits = 0;
 570   u32 drop = 0;
 571   u32 n_next_nodes;
 572   u64 time_in_policer_periods;
 573
 574   time_in_policer_periods =
 575     clib_cpu_time_now() >> POLICER_TICKS_PER_PERIOD_SHIFT;
 576
 577   n_next_nodes = node->n_next_nodes;
 578
 579   from = vlib_frame_vector_args (frame);
 580   n_left_from = frame->n_vectors;
 581
 582   /* First pass: compute hashes */
 583   while (n_left_from > 2)
 584     {
 585       vlib_buffer_t * b0, * b1;
 586       u32 bi0, bi1;
 587       u8 * h0, * h1;
 588       u32 sw_if_index0, sw_if_index1;
 589       u32 table_index0, table_index1;
 590       vnet_classify_table_t * t0, * t1;
 591
 592       /* Prefetch next iteration */
 593       {
 594         vlib_buffer_t * p1, * p2;
 595
 596         p1 = vlib_get_buffer (vm, from[1]);
 597         p2 = vlib_get_buffer (vm, from[2]);
 598
 599         vlib_prefetch_buffer_header (p1, STORE);
 600         CLIB_PREFETCH (p1->data, CLIB_CACHE_LINE_BYTES, STORE);
 601         vlib_prefetch_buffer_header (p2, STORE);
 602         CLIB_PREFETCH (p2->data, CLIB_CACHE_LINE_BYTES, STORE);
 603       }
 604
 605       bi0 = from[0];
 606       b0 = vlib_get_buffer (vm, bi0);
 607       h0 = b0->data;
 608
 609       bi1 = from[1];
 610       b1 = vlib_get_buffer (vm, bi1);
 611       h1 = b1->data;
 612
 613       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 614       table_index0 = pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 615
 616       sw_if_index1 = vnet_buffer (b1)->sw_if_index[VLIB_RX];
 617       table_index1 = pcm->classify_table_index_by_sw_if_index[tid][sw_if_index1];
 618
 619       t0 = pool_elt_at_index (vcm->tables, table_index0);
 620
 621       t1 = pool_elt_at_index (vcm->tables, table_index1);
 622
 623       vnet_buffer(b0)->l2_classify.hash =
 624         vnet_classify_hash_packet (t0, (u8 *) h0);
 625
 626       vnet_classify_prefetch_bucket (t0, vnet_buffer(b0)->l2_classify.hash);
 627
 628       vnet_buffer(b1)->l2_classify.hash =
 629         vnet_classify_hash_packet (t1, (u8 *) h1);
 630
 631       vnet_classify_prefetch_bucket (t1, vnet_buffer(b1)->l2_classify.hash);
 632
 633       vnet_buffer(b0)->l2_classify.table_index = table_index0;
 634
 635       vnet_buffer(b1)->l2_classify.table_index = table_index1;
 636
 637       from += 2;
 638       n_left_from -= 2;
 639     }
 640
 641   while (n_left_from > 0)
 642     {
 643       vlib_buffer_t * b0;
 644       u32 bi0;
 645       u8 * h0;
 646       u32 sw_if_index0;
 647       u32 table_index0;
 648       vnet_classify_table_t * t0;
 649
 650       bi0 = from[0];
 651       b0 = vlib_get_buffer (vm, bi0);
 652       h0 = b0->data;
 653
 654       sw_if_index0 = vnet_buffer (b0)->sw_if_index[VLIB_RX];
 655       table_index0 = pcm->classify_table_index_by_sw_if_index[tid][sw_if_index0];
 656
 657       t0 = pool_elt_at_index (vcm->tables, table_index0);
 658       vnet_buffer(b0)->l2_classify.hash =
 659         vnet_classify_hash_packet (t0, (u8 *) h0);
 660
 661       vnet_buffer(b0)->l2_classify.table_index = table_index0;
 662       vnet_classify_prefetch_bucket (t0, vnet_buffer(b0)->l2_classify.hash);
 663
 664       from++;
 665       n_left_from--;
 666     }
 667
 668   next_index = node->cached_next_index;
 669   from = vlib_frame_vector_args (frame);
 670   n_left_from = frame->n_vectors;
 671
 672   while (n_left_from > 0)
 673     {
 674       u32 n_left_to_next;
 675
 676       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 677
 678       /* Not enough load/store slots to dual loop... */
 679       while (n_left_from > 0 && n_left_to_next > 0)
 680         {
 681           u32 bi0;
 682           vlib_buffer_t * b0;
 683           u32 next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 684           u32 table_index0;
 685           vnet_classify_table_t * t0;
 686           vnet_classify_entry_t * e0;
 687           u64 hash0;
 688           u8 * h0;
 689           u8 act0;
 690
 691           /* Stride 3 seems to work best */
 692           if (PREDICT_TRUE (n_left_from > 3))
 693             {
 694               vlib_buffer_t * p1 = vlib_get_buffer(vm, from[3]);
 695               vnet_classify_table_t * tp1;
 696               u32 table_index1;
 697               u64 phash1;
 698
 699               table_index1 = vnet_buffer(p1)->l2_classify.table_index;
 700
 701               if (PREDICT_TRUE (table_index1 != ~0))
 702                 {
 703                   tp1 = pool_elt_at_index (vcm->tables, table_index1);
 704                   phash1 = vnet_buffer(p1)->l2_classify.hash;
 705                   vnet_classify_prefetch_entry (tp1, phash1);
 706                 }
 707             }
 708
 709           /* Speculatively enqueue b0 to the current next frame */
 710           bi0 = from[0];
 711           to_next[0] = bi0;
 712           from += 1;
 713           to_next += 1;
 714           n_left_from -= 1;
 715           n_left_to_next -= 1;
 716
 717           b0 = vlib_get_buffer (vm, bi0);
 718           h0 = b0->data;
 719           table_index0 = vnet_buffer(b0)->l2_classify.table_index;
 720           e0 = 0;
 721           t0 = 0;
 722
 723           if (tid == POLICER_CLASSIFY_TABLE_L2)
 724             {
 725               /* Feature bitmap update */
 726               vnet_buffer(b0)->l2.feature_bitmap &= ~L2INPUT_FEAT_POLICER_CLAS;
 727               /* Determine the next node */
 728               next0 = feat_bitmap_get_next_node_index(pcm->feat_next_node_index,
 729                 vnet_buffer(b0)->l2.feature_bitmap);
 730             }
 731           else
 732             vnet_get_config_data (pcm->vnet_config_main[tid],
 733                                   &b0->current_config_index,
 734                                   &next0,
 735                                   /* # bytes of config data */ 0);
 736
 737           vnet_buffer(b0)->l2_classify.opaque_index = ~0;
 738
 739           if (PREDICT_TRUE(table_index0 != ~0))
 740             {
 741               hash0 = vnet_buffer(b0)->l2_classify.hash;
 742               t0 = pool_elt_at_index (vcm->tables, table_index0);
 743               e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 744
 745               if (e0)
 746                 {
 747                   act0 = vnet_policer_police(vm,
 748                                              b0,
 749                                              e0->next_index,
 750                                              time_in_policer_periods,
 751                                              e0->opaque_index);
 752                   if (PREDICT_FALSE(act0 == SSE2_QOS_ACTION_DROP))
 753                     {
 754                       next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 755                       b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 756                       drop++;
 757                     }
 758                   hits++;
 759                 }
 760               else
 761                 {
 762                   while (1)
 763                     {
 764                       if (PREDICT_TRUE(t0->next_table_index != ~0))
 765                         {
 766                           t0 = pool_elt_at_index (vcm->tables,
 767                                                   t0->next_table_index);
 768                         }
 769                       else
 770                         {
 771                           next0 = (t0->miss_next_index < n_next_nodes)?
 772                                    t0->miss_next_index:next0;
 773                           misses++;
 774                           break;
 775                         }
 776
 777                       hash0 = vnet_classify_hash_packet (t0, (u8 *) h0);
 778                       e0 = vnet_classify_find_entry (t0, (u8 *) h0, hash0, now);
 779                       if (e0)
 780                         {
 781                           act0 = vnet_policer_police(vm,
 782                                                      b0,
 783                                                      e0->next_index,
 784                                                      time_in_policer_periods,
 785                                                      e0->opaque_index);
 786                           if (PREDICT_FALSE(act0 == SSE2_QOS_ACTION_DROP))
 787                             {
 788                               next0 = POLICER_CLASSIFY_NEXT_INDEX_DROP;
 789                               b0->error = node->errors[POLICER_CLASSIFY_ERROR_DROP];
 790                               drop++;
 791                             }
 792                           hits++;
 793                           chain_hits++;
 794                           break;
 795                         }
 796                     }
 797                 }
 798             }
 799           if (PREDICT_FALSE((node->flags & VLIB_NODE_FLAG_TRACE)
 800                             && (b0->flags & VLIB_BUFFER_IS_TRACED)))
 801             {
 802               policer_classify_trace_t * t =
 803                 vlib_add_trace (vm, node, b0, sizeof (*t));
 804               t->sw_if_index = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 805               t->next_index = next0;
 806               t->table_index = t0 ? t0 - vcm->tables : ~0;
 807               t->offset = e0 ? vnet_classify_get_offset (t0, e0): ~0;
 808               t->policer_index = e0 ? e0->next_index: ~0;
 809             }
 810
 811           /* Verify speculative enqueue, maybe switch current next frame */
 812           vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
 813                                            n_left_to_next, bi0, next0);
 814         }
 815
 816       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 817     }
 818
 819   vlib_node_increment_counter (vm, node->node_index,
 820                                POLICER_CLASSIFY_ERROR_MISS,
 821                                misses);
 822   vlib_node_increment_counter (vm, node->node_index,
 823                                POLICER_CLASSIFY_ERROR_HIT,
 824                                hits);
 825   vlib_node_increment_counter (vm, node->node_index,
 826                                POLICER_CLASSIFY_ERROR_CHAIN_HIT,
 827                                chain_hits);
 828   vlib_node_increment_counter (vm, node->node_index,
 829                                POLICER_CLASSIFY_ERROR_DROP,
 830                                drop);
 831
 832   return frame->n_vectors;
 833 }
 834
 835 static uword
 836 ip4_policer_classify (vlib_main_t * vm,
 837                       vlib_node_runtime_t * node,
 838                       vlib_frame_t * frame)
 839 {
 840   return policer_classify_inline(vm, node, frame, POLICER_CLASSIFY_TABLE_IP4);
 841 }
 842
 843 VLIB_REGISTER_NODE (ip4_policer_classify_node) = {
 844   .function = ip4_policer_classify,
 845   .name = "ip4-policer-classify",
 846   .vector_size = sizeof (u32),
 847   .format_trace = format_policer_classify_trace,
 848   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 849   .error_strings = policer_classify_error_strings,
 850   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 851   .next_nodes = {
 852     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 853   },
 854 };
 855
 856 VLIB_NODE_FUNCTION_MULTIARCH (ip4_policer_classify_node, ip4_policer_classify);
 857
 858 static uword
 859 ip6_policer_classify (vlib_main_t * vm,
 860                       vlib_node_runtime_t * node,
 861                       vlib_frame_t * frame)
 862 {
 863   return policer_classify_inline(vm, node, frame, POLICER_CLASSIFY_TABLE_IP6);
 864 }
 865
 866 VLIB_REGISTER_NODE (ip6_policer_classify_node) = {
 867   .function = ip6_policer_classify,
 868   .name = "ip6-policer-classify",
 869   .vector_size = sizeof (u32),
 870   .format_trace = format_policer_classify_trace,
 871   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 872   .error_strings = policer_classify_error_strings,
 873   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 874   .next_nodes = {
 875     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 876   },
 877 };
 878
 879 VLIB_NODE_FUNCTION_MULTIARCH (ip6_policer_classify_node, ip6_policer_classify);
 880
 881 static uword
 882 l2_policer_classify (vlib_main_t * vm,
 883                      vlib_node_runtime_t * node,
 884                      vlib_frame_t * frame)
 885 {
 886   return policer_classify_inline(vm, node, frame, POLICER_CLASSIFY_TABLE_L2);
 887 }
 888
 889 VLIB_REGISTER_NODE (l2_policer_classify_node) = {
 890   .function = l2_policer_classify,
 891   .name = "l2-policer-classify",
 892   .vector_size = sizeof (u32),
 893   .format_trace = format_policer_classify_trace,
 894   .n_errors = ARRAY_LEN(policer_classify_error_strings),
 895   .error_strings = policer_classify_error_strings,
 896   .n_next_nodes = POLICER_CLASSIFY_NEXT_INDEX_N_NEXT,
 897   .next_nodes = {
 898     [POLICER_CLASSIFY_NEXT_INDEX_DROP] = "error-drop",
 899   },
 900 };
 901
 902 VLIB_NODE_FUNCTION_MULTIARCH (l2_policer_classify_node, l2_policer_classify);
 903
 904
 905 static clib_error_t *
 906 policer_classify_init (vlib_main_t *vm)
 907 {
 908   policer_classify_main_t * pcm = &policer_classify_main;
 909
 910   pcm->vlib_main = vm;
 911   pcm->vnet_main = vnet_get_main();
 912   pcm->vnet_classify_main = &vnet_classify_main;
 913
 914   /* Initialize L2 feature next-node indexes */
 915   feat_bitmap_init_next_nodes(vm,
 916                               l2_policer_classify_node.index,
 917                               L2INPUT_N_FEAT,
 918                               l2input_get_feat_names(),
 919                               pcm->feat_next_node_index);
 920
 921   return 0;
 922 }
 923
 924 VLIB_INIT_FUNCTION (policer_classify_init);