vnet/vnet/devices/dpdk/node.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <vnet/vnet.h>
  16 #include <vppinfra/vec.h>
  17 #include <vppinfra/error.h>
  18 #include <vppinfra/format.h>
  19 #include <vppinfra/xxhash.h>
  20
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <vnet/devices/dpdk/dpdk.h>
  23 #include <vnet/classify/vnet_classify.h>
  24 #include <vnet/mpls-gre/packet.h>
  25
  26 #include "dpdk_priv.h"
  27
  28 #ifndef MAX
  29 #define MAX(a,b) ((a) < (b) ? (b) : (a))
  30 #endif
  31
  32 #ifndef MIN
  33 #define MIN(a,b) ((a) < (b) ? (a) : (b))
  34 #endif
  35
  36 /*
  37  * At least in certain versions of ESXi, vmware e1000's don't honor the
  38  * "strip rx CRC" bit. Set this flag to work around that bug FOR UNIT TEST ONLY.
  39  *
  40  * If wireshark complains like so:
  41  *
  42  * "Frame check sequence: 0x00000000 [incorrect, should be <hex-num>]"
  43  * and you're using ESXi emulated e1000's, set this flag FOR UNIT TEST ONLY.
  44  *
  45  * Note: do NOT check in this file with this workaround enabled! You'll lose
  46  * actual data from e.g. 10xGE interfaces. The extra 4 bytes annoy
  47  * wireshark, but they're harmless...
  48  */
  49 #define VMWARE_LENGTH_BUG_WORKAROUND 0
  50
  51 typedef struct {
  52   u32 cached_next_index;
  53
  54   /* convenience variables */
  55   vlib_main_t * vlib_main;
  56   vnet_main_t * vnet_main;
  57 } handoff_dispatch_main_t;
  58
  59 typedef struct {
  60   u32 buffer_index;
  61   u32 next_index;
  62   u32 sw_if_index;
  63 } handoff_dispatch_trace_t;
  64
  65 /* packet trace format function */
  66 static u8 * format_handoff_dispatch_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   handoff_dispatch_trace_t * t = va_arg (*args, handoff_dispatch_trace_t *);
  71
  72   s = format (s, "HANDOFF_DISPATCH: sw_if_index %d next_index %d buffer 0x%x",
  73       t->sw_if_index,
  74       t->next_index,
  75       t->buffer_index);
  76   return s;
  77 }
  78
  79 handoff_dispatch_main_t handoff_dispatch_main;
  80
  81 vlib_node_registration_t handoff_dispatch_node;
  82
  83 #define foreach_handoff_dispatch_error \
  84 _(EXAMPLE, "example packets")
  85
  86 typedef enum {
  87 #define _(sym,str) HANDOFF_DISPATCH_ERROR_##sym,
  88   foreach_handoff_dispatch_error
  89 #undef _
  90   HANDOFF_DISPATCH_N_ERROR,
  91 } handoff_dispatch_error_t;
  92
  93 static char * handoff_dispatch_error_strings[] = {
  94 #define _(sym,string) string,
  95   foreach_handoff_dispatch_error
  96 #undef _
  97 };
  98
  99 static inline
 100 void vlib_put_handoff_queue_elt (vlib_frame_queue_elt_t * hf)
 101 {
 102   CLIB_MEMORY_BARRIER();
 103   hf->valid = 1;
 104 }
 105
 106 static uword
 107 handoff_dispatch_node_fn (vlib_main_t * vm,
 108                   vlib_node_runtime_t * node,
 109                   vlib_frame_t * frame)
 110 {
 111   u32 n_left_from, * from, * to_next;
 112   dpdk_rx_next_t next_index;
 113
 114   from = vlib_frame_vector_args (frame);
 115   n_left_from = frame->n_vectors;
 116   next_index = node->cached_next_index;
 117
 118   while (n_left_from > 0)
 119     {
 120       u32 n_left_to_next;
 121
 122       vlib_get_next_frame (vm, node, next_index,
 123                            to_next, n_left_to_next);
 124
 125       while (n_left_from >= 4 && n_left_to_next >= 2)
 126         {
 127           u32 bi0, bi1;
 128           vlib_buffer_t * b0, * b1;
 129           u32 next0, next1;
 130           u32 sw_if_index0, sw_if_index1;
 131
 132           /* Prefetch next iteration. */
 133           {
 134             vlib_buffer_t * p2, * p3;
 135
 136             p2 = vlib_get_buffer (vm, from[2]);
 137             p3 = vlib_get_buffer (vm, from[3]);
 138
 139             vlib_prefetch_buffer_header (p2, LOAD);
 140             vlib_prefetch_buffer_header (p3, LOAD);
 141           }
 142
 143           /* speculatively enqueue b0 and b1 to the current next frame */
 144           to_next[0] = bi0 = from[0];
 145           to_next[1] = bi1 = from[1];
 146           from += 2;
 147           to_next += 2;
 148           n_left_from -= 2;
 149           n_left_to_next -= 2;
 150
 151           b0 = vlib_get_buffer (vm, bi0);
 152           b1 = vlib_get_buffer (vm, bi1);
 153
 154           next0 = vnet_buffer(b0)->io_handoff.next_index;
 155           next1 = vnet_buffer(b1)->io_handoff.next_index;
 156
 157           if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
 158             {
 159             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 160               {
 161                 vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
 162                 handoff_dispatch_trace_t *t =
 163                   vlib_add_trace (vm, node, b0, sizeof (*t));
 164                 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 165                 t->sw_if_index = sw_if_index0;
 166                 t->next_index = next0;
 167                 t->buffer_index = bi0;
 168               }
 169             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 170               {
 171                 vlib_trace_buffer (vm, node, next1, b1, /* follow_chain */ 0);
 172                 handoff_dispatch_trace_t *t =
 173                   vlib_add_trace (vm, node, b1, sizeof (*t));
 174                 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 175                 t->sw_if_index = sw_if_index1;
 176                 t->next_index = next1;
 177                 t->buffer_index = bi1;
 178               }
 179             }
 180
 181           /* verify speculative enqueues, maybe switch current next frame */
 182           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 183                                            to_next, n_left_to_next,
 184                                            bi0, bi1, next0, next1);
 185         }
 186
 187       while (n_left_from > 0 && n_left_to_next > 0)
 188         {
 189           u32 bi0;
 190           vlib_buffer_t * b0;
 191           u32 next0;
 192           u32 sw_if_index0;
 193
 194           /* speculatively enqueue b0 to the current next frame */
 195           bi0 = from[0];
 196           to_next[0] = bi0;
 197           from += 1;
 198           to_next += 1;
 199           n_left_from -= 1;
 200           n_left_to_next -= 1;
 201
 202           b0 = vlib_get_buffer (vm, bi0);
 203
 204           next0 = vnet_buffer(b0)->io_handoff.next_index;
 205
 206           if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
 207             {
 208             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 209               {
 210                 vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
 211                 handoff_dispatch_trace_t *t =
 212                   vlib_add_trace (vm, node, b0, sizeof (*t));
 213                 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 214                 t->sw_if_index = sw_if_index0;
 215                 t->next_index = next0;
 216                 t->buffer_index = bi0;
 217               }
 218             }
 219
 220           /* verify speculative enqueue, maybe switch current next frame */
 221           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 222                                            to_next, n_left_to_next,
 223                                            bi0, next0);
 224         }
 225
 226       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 227     }
 228
 229   return frame->n_vectors;
 230 }
 231
 232 VLIB_REGISTER_NODE (handoff_dispatch_node) = {
 233   .function = handoff_dispatch_node_fn,
 234   .name = "handoff-dispatch",
 235   .vector_size = sizeof (u32),
 236   .format_trace = format_handoff_dispatch_trace,
 237   .type = VLIB_NODE_TYPE_INTERNAL,
 238   .flags = VLIB_NODE_FLAG_IS_HANDOFF,
 239
 240   .n_errors = ARRAY_LEN(handoff_dispatch_error_strings),
 241   .error_strings = handoff_dispatch_error_strings,
 242
 243   .n_next_nodes = DPDK_RX_N_NEXT,
 244
 245   .next_nodes = {
 246         [DPDK_RX_NEXT_DROP] = "error-drop",
 247         [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
 248         [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input",
 249         [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
 250         [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
 251   },
 252 };
 253
 254 VLIB_NODE_FUNCTION_MULTIARCH (handoff_dispatch_node, handoff_dispatch_node_fn)
 255
 256 clib_error_t *handoff_dispatch_init (vlib_main_t *vm)
 257 {
 258   handoff_dispatch_main_t * mp = &handoff_dispatch_main;
 259
 260   mp->vlib_main = vm;
 261   mp->vnet_main = &vnet_main;
 262
 263   return 0;
 264 }
 265
 266 VLIB_INIT_FUNCTION (handoff_dispatch_init);
 267
 268 u32 dpdk_get_handoff_node_index (void)
 269 {
 270   return handoff_dispatch_node.index;
 271 }
 272
 273 static char * dpdk_error_strings[] = {
 274 #define _(n,s) s,
 275     foreach_dpdk_error
 276 #undef _
 277 };
 278
 279 always_inline void
 280 dpdk_rx_next_and_error_from_mb_flags_x1 (dpdk_device_t *xd, struct rte_mbuf *mb,
 281                                          vlib_buffer_t *b0,
 282                                          u8 * next0, u8 * error0)
 283 {
 284   u8 is0_ip4, is0_ip6, is0_mpls, n0;
 285   uint16_t mb_flags = mb->ol_flags;
 286
 287   if (PREDICT_FALSE(mb_flags & (
 288 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 289        PKT_EXT_RX_PKT_ERROR | PKT_EXT_RX_BAD_FCS   |
 290 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 291         PKT_RX_IP_CKSUM_BAD  | PKT_RX_L4_CKSUM_BAD
 292     )))
 293     {
 294       /* some error was flagged. determine the drop reason */
 295       n0 = DPDK_RX_NEXT_DROP;
 296       *error0 =
 297 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 298         (mb_flags & PKT_EXT_RX_PKT_ERROR) ? DPDK_ERROR_RX_PACKET_ERROR :
 299         (mb_flags & PKT_EXT_RX_BAD_FCS) ? DPDK_ERROR_RX_BAD_FCS :
 300 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 301         (mb_flags & PKT_RX_IP_CKSUM_BAD) ? DPDK_ERROR_IP_CHECKSUM_ERROR :
 302         (mb_flags & PKT_RX_L4_CKSUM_BAD) ? DPDK_ERROR_L4_CHECKSUM_ERROR :
 303         DPDK_ERROR_NONE;
 304     }
 305   else
 306     {
 307       *error0 = DPDK_ERROR_NONE;
 308       if (xd->per_interface_next_index != ~0)
 309         n0 = xd->per_interface_next_index;
 310       else if (mb_flags & PKT_RX_VLAN_PKT)
 311         n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
 312       else
 313         {
 314           n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
 315 #if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
 316           is0_ip4 = RTE_ETH_IS_IPV4_HDR(mb->packet_type) != 0;
 317 #else
 318           is0_ip4 = (mb_flags & (PKT_RX_IPV4_HDR | PKT_RX_IPV4_HDR_EXT)) != 0;
 319 #endif
 320
 321           if (PREDICT_TRUE(is0_ip4))
 322             n0 = DPDK_RX_NEXT_IP4_INPUT;
 323           else
 324             {
 325 #if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
 326               is0_ip6 = RTE_ETH_IS_IPV6_HDR(mb->packet_type) != 0;
 327 #else
 328               is0_ip6 =
 329                       (mb_flags & (PKT_RX_IPV6_HDR | PKT_RX_IPV6_HDR_EXT)) != 0;
 330 #endif
 331               if (PREDICT_TRUE(is0_ip6))
 332                 n0 = DPDK_RX_NEXT_IP6_INPUT;
 333               else
 334                 {
 335                   ethernet_header_t *h0 = (ethernet_header_t *) b0->data;
 336                   is0_mpls = (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST));
 337                   n0 = is0_mpls ? DPDK_RX_NEXT_MPLS_INPUT : n0;
 338                 }
 339             }
 340         }
 341     }
 342   *next0 = n0;
 343 }
 344
 345 void dpdk_rx_trace (dpdk_main_t * dm,
 346                     vlib_node_runtime_t * node,
 347                     dpdk_device_t * xd,
 348                     u16 queue_id,
 349                     u32 * buffers,
 350                     uword n_buffers)
 351 {
 352   vlib_main_t * vm = vlib_get_main();
 353   u32 * b, n_left;
 354   u8 next0;
 355
 356   n_left = n_buffers;
 357   b = buffers;
 358
 359   while (n_left >= 1)
 360     {
 361       u32 bi0;
 362       vlib_buffer_t * b0;
 363       dpdk_rx_dma_trace_t * t0;
 364       struct rte_mbuf *mb;
 365       u8 error0;
 366
 367       bi0 = b[0];
 368       n_left -= 1;
 369
 370       b0 = vlib_get_buffer (vm, bi0);
 371       mb = rte_mbuf_from_vlib_buffer(b0);
 372       dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
 373                                                &next0, &error0);
 374       vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
 375       t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 376       t0->queue_index = queue_id;
 377       t0->device_index = xd->device_index;
 378       t0->buffer_index = bi0;
 379
 380       clib_memcpy (&t0->mb, mb, sizeof (t0->mb));
 381       clib_memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
 382       clib_memcpy (t0->buffer.pre_data, b0->data, sizeof (t0->buffer.pre_data));
 383
 384 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 385       /*
 386        * Clear overloaded TX offload flags when a DPDK driver
 387        * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
 388        */
 389       mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
 390 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 391
 392       b += 1;
 393     }
 394 }
 395
 396 /*
 397  * dpdk_efd_update_counters()
 398  * Update EFD (early-fast-discard) counters
 399  */
 400 void dpdk_efd_update_counters (dpdk_device_t *xd,
 401                                u32 n_buffers,
 402                                u16 enabled)
 403 {
 404   if (enabled & DPDK_EFD_MONITOR_ENABLED)
 405     {
 406       u64 now = clib_cpu_time_now();
 407       if (xd->efd_agent.last_poll_time > 0)
 408         {
 409           u64 elapsed_time = (now - xd->efd_agent.last_poll_time);
 410           if (elapsed_time > xd->efd_agent.max_poll_delay)
 411             xd->efd_agent.max_poll_delay = elapsed_time;
 412         }
 413       xd->efd_agent.last_poll_time = now;
 414     }
 415
 416   xd->efd_agent.total_packet_cnt += n_buffers;
 417   xd->efd_agent.last_burst_sz = n_buffers;
 418
 419   if (n_buffers > xd->efd_agent.max_burst_sz)
 420     xd->efd_agent.max_burst_sz = n_buffers;
 421
 422   if (PREDICT_FALSE(n_buffers == VLIB_FRAME_SIZE))
 423     {
 424       xd->efd_agent.full_frames_cnt++;
 425       xd->efd_agent.consec_full_frames_cnt++;
 426     }
 427   else
 428     {
 429       xd->efd_agent.consec_full_frames_cnt = 0;
 430     }
 431 }
 432
 433 /* is_efd_discardable()
 434  *   returns non zero DPDK error if packet meets early-fast-discard criteria,
 435  *           zero otherwise
 436  */
 437 u32 is_efd_discardable (vlib_thread_main_t *tm,
 438                         vlib_buffer_t * b0,
 439                         struct rte_mbuf *mb)
 440 {
 441   ethernet_header_t *eh = (ethernet_header_t *) b0->data;
 442
 443   if (eh->type == clib_host_to_net_u16(ETHERNET_TYPE_IP4))
 444     {
 445       ip4_header_t *ipv4 =
 446           (ip4_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 447       u8 pkt_prec = (ipv4->tos >> 5);
 448
 449       return (tm->efd.ip_prec_bitmap & (1 << pkt_prec) ?
 450                   DPDK_ERROR_IPV4_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 451     }
 452   else if (eh->type == clib_net_to_host_u16(ETHERNET_TYPE_IP6))
 453     {
 454       ip6_header_t *ipv6 =
 455           (ip6_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 456       u8 pkt_tclass =
 457           ((ipv6->ip_version_traffic_class_and_flow_label >> 20) & 0xff);
 458
 459       return (tm->efd.ip_prec_bitmap & (1 << pkt_tclass) ?
 460                   DPDK_ERROR_IPV6_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 461     }
 462   else if (eh->type == clib_net_to_host_u16(ETHERNET_TYPE_MPLS_UNICAST))
 463     {
 464       mpls_unicast_header_t *mpls =
 465           (mpls_unicast_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 466       u8 pkt_exp = ((mpls->label_exp_s_ttl >> 9) & 0x07);
 467
 468       return (tm->efd.mpls_exp_bitmap & (1 << pkt_exp) ?
 469                   DPDK_ERROR_MPLS_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 470     }
 471   else if ((eh->type == clib_net_to_host_u16(ETHERNET_TYPE_VLAN)) ||
 472            (eh->type == clib_net_to_host_u16(ETHERNET_TYPE_DOT1AD)))
 473     {
 474       ethernet_vlan_header_t *vlan =
 475           (ethernet_vlan_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 476       u8 pkt_cos = ((vlan->priority_cfi_and_id >> 13) & 0x07);
 477
 478       return (tm->efd.vlan_cos_bitmap & (1 << pkt_cos) ?
 479                   DPDK_ERROR_VLAN_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 480     }
 481
 482   return DPDK_ERROR_NONE;
 483 }
 484
 485 /*
 486  * This function is used when there are no worker threads.
 487  * The main thread performs IO and forwards the packets.
 488  */
 489 static inline u32 dpdk_device_input ( dpdk_main_t * dm,
 490                                       dpdk_device_t * xd,
 491                                       vlib_node_runtime_t * node,
 492                                       u32 cpu_index,
 493                                       u16 queue_id,
 494                                       int use_efd)
 495 {
 496   u32 n_buffers;
 497   u32 next_index = DPDK_RX_NEXT_ETHERNET_INPUT;
 498   u32 n_left_to_next, * to_next;
 499   u32 mb_index;
 500   vlib_main_t * vm = vlib_get_main();
 501   uword n_rx_bytes = 0;
 502   u32 n_trace, trace_cnt __attribute__((unused));
 503   vlib_buffer_free_list_t * fl;
 504   u8 efd_discard_burst = 0;
 505   u16 ip_align_offset = 0;
 506   u32 buffer_flags_template;
 507
 508   if (xd->admin_up == 0)
 509     return 0;
 510
 511   n_buffers = dpdk_rx_burst(dm, xd, queue_id);
 512
 513   if (n_buffers == 0)
 514     {
 515       /* check if EFD (dpdk) is enabled */
 516       if (PREDICT_FALSE(use_efd && dm->efd.enabled))
 517         {
 518           /* reset a few stats */
 519           xd->efd_agent.last_poll_time = 0;
 520           xd->efd_agent.last_burst_sz = 0;
 521         }
 522       return 0;
 523     }
 524
 525   if (xd->pmd == VNET_DPDK_PMD_THUNDERX)
 526       ip_align_offset = 6;
 527
 528   buffer_flags_template = dm->buffer_flags_template;
 529
 530   vec_reset_length (xd->d_trace_buffers);
 531   trace_cnt = n_trace = vlib_get_trace_count (vm, node);
 532
 533   fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
 534
 535   /*
 536    * DAW-FIXME: VMXNET3 device stop/start doesn't work,
 537    * therefore fake the stop in the dpdk driver by
 538    * silently dropping all of the incoming pkts instead of
 539    * stopping the driver / hardware.
 540    */
 541   if (PREDICT_FALSE(xd->admin_up != 1))
 542     {
 543       for (mb_index = 0; mb_index < n_buffers; mb_index++)
 544         rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]);
 545
 546       return 0;
 547     }
 548
 549   /* Check for congestion if EFD (Early-Fast-Discard) is enabled
 550    * in any mode (e.g. dpdk, monitor, or drop_all)
 551    */
 552   if (PREDICT_FALSE(use_efd && dm->efd.enabled))
 553     {
 554       /* update EFD counters */
 555       dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
 556
 557       if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED))
 558         {
 559           /* discard all received packets */
 560           for (mb_index = 0; mb_index < n_buffers; mb_index++)
 561             rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]);
 562
 563           xd->efd_agent.discard_cnt += n_buffers;
 564           increment_efd_drop_counter(vm,
 565                                      DPDK_ERROR_VLAN_EFD_DROP_PKTS,
 566                                      n_buffers);
 567
 568           return 0;
 569         }
 570
 571       if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >=
 572                         dm->efd.consec_full_frames_hi_thresh))
 573         {
 574           u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index,
 575                                                        queue_id);
 576           if (device_queue_sz >= dm->efd.queue_hi_thresh)
 577             {
 578               /* dpdk device queue has reached the critical threshold */
 579               xd->efd_agent.congestion_cnt++;
 580
 581               /* apply EFD to packets from the burst */
 582               efd_discard_burst = 1;
 583             }
 584         }
 585     }
 586
 587   mb_index = 0;
 588
 589   while (n_buffers > 0)
 590     {
 591       u32 bi0;
 592       u8 next0, error0;
 593       u32 l3_offset0;
 594       vlib_buffer_t * b0, * b_seg, * b_chain = 0;
 595       u32 cntr_type;
 596
 597       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 598
 599       while (n_buffers > 0 && n_left_to_next > 0)
 600         {
 601           u8 nb_seg = 1;
 602           struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index];
 603           struct rte_mbuf *mb_seg = mb->next;
 604
 605           if (PREDICT_TRUE(n_buffers > 2))
 606           {
 607               struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
 608               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
 609               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, STORE);
 610               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
 611           }
 612
 613           ASSERT(mb);
 614
 615           b0 = vlib_buffer_from_rte_mbuf(mb);
 616
 617           /* check whether EFD is looking for packets to discard */
 618           if (PREDICT_FALSE(efd_discard_burst))
 619             {
 620               vlib_thread_main_t * tm = vlib_get_thread_main();
 621
 622               if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
 623                 {
 624                   rte_pktmbuf_free(mb);
 625                   xd->efd_agent.discard_cnt++;
 626                   increment_efd_drop_counter(vm,
 627                                              cntr_type,
 628                                              1);
 629                   n_buffers--;
 630                   mb_index++;
 631                   continue;
 632                 }
 633             }
 634
 635           /* Prefetch one next segment if it exists. */
 636           if (PREDICT_FALSE(mb->nb_segs > 1))
 637             {
 638               struct rte_mbuf *pfmb = mb->next;
 639               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
 640               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
 641               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
 642               b_chain = b0;
 643             }
 644
 645           vlib_buffer_init_for_free_list (b0, fl);
 646           b0->clone_count = 0;
 647
 648           bi0 = vlib_get_buffer_index (vm, b0);
 649
 650           to_next[0] = bi0;
 651           to_next++;
 652           n_left_to_next--;
 653
 654           dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
 655                                                    &next0, &error0);
 656 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 657           /*
 658            * Clear overloaded TX offload flags when a DPDK driver
 659            * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
 660            */
 661
 662           if (PREDICT_TRUE(trace_cnt == 0))
 663             mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
 664           else
 665             trace_cnt--;
 666 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 667
 668           b0->error = node->errors[error0];
 669
 670           l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT ||
 671                          next0 == DPDK_RX_NEXT_IP6_INPUT ||
 672                          next0 == DPDK_RX_NEXT_MPLS_INPUT) ?
 673                         sizeof (ethernet_header_t) : 0);
 674
 675           b0->current_data = l3_offset0;
 676           b0->current_length = mb->data_len - l3_offset0;
 677
 678           if (PREDICT_FALSE (ip_align_offset != 0))
 679             {
 680               if (next0 == DPDK_RX_NEXT_IP4_INPUT ||
 681                   next0 == DPDK_RX_NEXT_IP6_INPUT)
 682                 b0->current_data += ip_align_offset;
 683             }
 684
 685           b0->flags = buffer_flags_template;
 686
 687           if (VMWARE_LENGTH_BUG_WORKAROUND)
 688               b0->current_length -= 4;
 689
 690           vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
 691           vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
 692           n_rx_bytes += mb->pkt_len;
 693
 694           /* Process subsequent segments of multi-segment packets */
 695           while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
 696             {
 697               ASSERT(mb_seg != 0);
 698
 699               b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
 700               vlib_buffer_init_for_free_list (b_seg, fl);
 701               b_seg->clone_count = 0;
 702
 703               ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
 704               ASSERT(b_seg->current_data == 0);
 705
 706               /*
 707                * The driver (e.g. virtio) may not put the packet data at the start
 708                * of the segment, so don't assume b_seg->current_data == 0 is correct.
 709                */
 710               b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data;
 711
 712               b_seg->current_length = mb_seg->data_len;
 713               b0->total_length_not_including_first_buffer +=
 714                 mb_seg->data_len;
 715
 716               b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
 717               b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
 718
 719               b_chain = b_seg;
 720               mb_seg = mb_seg->next;
 721               nb_seg++;
 722             }
 723
 724           /*
 725            * Turn this on if you run into
 726            * "bad monkey" contexts, and you want to know exactly
 727            * which nodes they've visited... See main.c...
 728            */
 729           VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0);
 730
 731           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 732                                            to_next, n_left_to_next,
 733                                            bi0, next0);
 734           if (PREDICT_FALSE (n_trace > mb_index))
 735             vec_add1 (xd->d_trace_buffers, bi0);
 736           n_buffers--;
 737           mb_index++;
 738         }
 739       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 740     }
 741
 742   if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0))
 743     {
 744       dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers,
 745                      vec_len (xd->d_trace_buffers));
 746       vlib_set_trace_count (vm, node, n_trace - vec_len (xd->d_trace_buffers));
 747     }
 748
 749   vlib_increment_combined_counter
 750     (vnet_get_main()->interface_main.combined_sw_if_counters
 751      + VNET_INTERFACE_COUNTER_RX,
 752      cpu_index,
 753      xd->vlib_sw_if_index,
 754      mb_index, n_rx_bytes);
 755
 756   dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index);
 757   dw->aggregate_rx_packets += mb_index;
 758
 759   return mb_index;
 760 }
 761
 762 static inline void poll_rate_limit(dpdk_main_t * dm)
 763 {
 764   /* Limit the poll rate by sleeping for N msec between polls */
 765   if (PREDICT_FALSE (dm->poll_sleep != 0))
 766   {
 767     struct timespec ts, tsrem;
 768
 769     ts.tv_sec = 0;
 770     ts.tv_nsec = 1000*1000*dm->poll_sleep; /* 1ms */
 771
 772     while (nanosleep(&ts, &tsrem) < 0)
 773       {
 774         ts = tsrem;
 775       }
 776   }
 777 }
 778
 779 static uword
 780 dpdk_input (vlib_main_t * vm,
 781             vlib_node_runtime_t * node,
 782             vlib_frame_t * f)
 783 {
 784   dpdk_main_t * dm = &dpdk_main;
 785   dpdk_device_t * xd;
 786   uword n_rx_packets = 0;
 787   dpdk_device_and_queue_t * dq;
 788   u32 cpu_index = os_get_cpu_number();
 789
 790   /*
 791    * Poll all devices on this cpu for input/interrupts.
 792    */
 793   vec_foreach (dq, dm->devices_by_cpu[cpu_index])
 794     {
 795       xd = vec_elt_at_index(dm->devices, dq->device);
 796       ASSERT(dq->queue_id == 0);
 797       n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, 0, 0);
 798     }
 799
 800   poll_rate_limit(dm);
 801
 802   return n_rx_packets;
 803 }
 804
 805 uword
 806 dpdk_input_rss (vlib_main_t * vm,
 807       vlib_node_runtime_t * node,
 808       vlib_frame_t * f)
 809 {
 810   dpdk_main_t * dm = &dpdk_main;
 811   dpdk_device_t * xd;
 812   uword n_rx_packets = 0;
 813   dpdk_device_and_queue_t * dq;
 814   u32 cpu_index = os_get_cpu_number();
 815
 816   /*
 817    * Poll all devices on this cpu for input/interrupts.
 818    */
 819   vec_foreach (dq, dm->devices_by_cpu[cpu_index])
 820     {
 821       xd = vec_elt_at_index(dm->devices, dq->device);
 822       n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, 0);
 823     }
 824
 825   poll_rate_limit(dm);
 826
 827   return n_rx_packets;
 828 }
 829
 830 uword
 831 dpdk_input_efd (vlib_main_t * vm,
 832       vlib_node_runtime_t * node,
 833       vlib_frame_t * f)
 834 {
 835   dpdk_main_t * dm = &dpdk_main;
 836   dpdk_device_t * xd;
 837   uword n_rx_packets = 0;
 838   dpdk_device_and_queue_t * dq;
 839   u32 cpu_index = os_get_cpu_number();
 840
 841   /*
 842    * Poll all devices on this cpu for input/interrupts.
 843    */
 844   vec_foreach (dq, dm->devices_by_cpu[cpu_index])
 845     {
 846       xd = vec_elt_at_index(dm->devices, dq->device);
 847       n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id, 1);
 848     }
 849
 850   poll_rate_limit(dm);
 851
 852   return n_rx_packets;
 853 }
 854
 855
 856 VLIB_REGISTER_NODE (dpdk_input_node) = {
 857   .function = dpdk_input,
 858   .type = VLIB_NODE_TYPE_INPUT,
 859   .name = "dpdk-input",
 860
 861   /* Will be enabled if/when hardware is detected. */
 862   .state = VLIB_NODE_STATE_DISABLED,
 863
 864   .format_buffer = format_ethernet_header_with_length,
 865   .format_trace = format_dpdk_rx_dma_trace,
 866
 867   .n_errors = DPDK_N_ERROR,
 868   .error_strings = dpdk_error_strings,
 869
 870   .n_next_nodes = DPDK_RX_N_NEXT,
 871   .next_nodes = {
 872     [DPDK_RX_NEXT_DROP] = "error-drop",
 873     [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
 874     [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
 875     [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
 876     [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
 877   },
 878 };
 879
 880
 881 /* handle dpdk_input_rss alternative function */
 882 VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input)
 883 VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input_rss)
 884 VLIB_NODE_FUNCTION_MULTIARCH_CLONE(dpdk_input_efd)
 885
 886 /* this macro defines dpdk_input_rss_multiarch_select() */
 887 CLIB_MULTIARCH_SELECT_FN(dpdk_input);
 888 CLIB_MULTIARCH_SELECT_FN(dpdk_input_rss);
 889 CLIB_MULTIARCH_SELECT_FN(dpdk_input_efd);
 890
 891 /*
 892  * Override the next nodes for the dpdk input nodes.
 893  * Must be invoked prior to VLIB_INIT_FUNCTION calls.
 894  */
 895 void dpdk_set_next_node (dpdk_rx_next_t next, char *name)
 896 {
 897   vlib_node_registration_t *r = &dpdk_input_node;
 898   vlib_node_registration_t *r_io = &dpdk_io_input_node;
 899   vlib_node_registration_t *r_handoff = &handoff_dispatch_node;
 900
 901   switch (next)
 902     {
 903     case DPDK_RX_NEXT_IP4_INPUT:
 904     case DPDK_RX_NEXT_IP6_INPUT:
 905     case DPDK_RX_NEXT_MPLS_INPUT:
 906     case DPDK_RX_NEXT_ETHERNET_INPUT:
 907       r->next_nodes[next] = name;
 908       r_io->next_nodes[next] = name;
 909       r_handoff->next_nodes[next] = name;
 910       break;
 911
 912     default:
 913       clib_warning ("%s: illegal next %d\n", __FUNCTION__, next);
 914       break;
 915     }
 916 }
 917
 918 inline vlib_frame_queue_elt_t *
 919 vlib_get_handoff_queue_elt (u32 vlib_worker_index)
 920 {
 921   vlib_frame_queue_t *fq;
 922   vlib_frame_queue_elt_t *elt;
 923   u64 new_tail;
 924
 925   fq = vlib_frame_queues[vlib_worker_index];
 926   ASSERT (fq);
 927
 928   new_tail = __sync_add_and_fetch (&fq->tail, 1);
 929
 930   /* Wait until a ring slot is available */
 931   while (new_tail >= fq->head_hint + fq->nelts)
 932       vlib_worker_thread_barrier_check ();
 933
 934   elt = fq->elts + (new_tail & (fq->nelts-1));
 935
 936   /* this would be very bad... */
 937   while (elt->valid)
 938     ;
 939
 940   elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME;
 941   elt->last_n_vectors = elt->n_vectors = 0;
 942
 943   return elt;
 944 }
 945
 946 static inline vlib_frame_queue_elt_t *
 947 dpdk_get_handoff_queue_elt (
 948     u32 vlib_worker_index,
 949     vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index)
 950 {
 951   vlib_frame_queue_elt_t *elt;
 952
 953   if (handoff_queue_elt_by_worker_index [vlib_worker_index])
 954       return handoff_queue_elt_by_worker_index [vlib_worker_index];
 955
 956   elt = vlib_get_handoff_queue_elt (vlib_worker_index);
 957
 958   handoff_queue_elt_by_worker_index [vlib_worker_index] = elt;
 959
 960   return elt;
 961 }
 962
 963 static inline vlib_frame_queue_t *
 964 is_vlib_handoff_queue_congested (
 965     u32 vlib_worker_index,
 966     u32 queue_hi_thresh,
 967     vlib_frame_queue_t ** handoff_queue_by_worker_index)
 968 {
 969   vlib_frame_queue_t *fq;
 970
 971   fq = handoff_queue_by_worker_index [vlib_worker_index];
 972   if (fq != (vlib_frame_queue_t *)(~0))
 973       return fq;
 974
 975   fq = vlib_frame_queues[vlib_worker_index];
 976   ASSERT (fq);
 977
 978   if (PREDICT_FALSE(fq->tail >= (fq->head_hint + queue_hi_thresh))) {
 979     /* a valid entry in the array will indicate the queue has reached
 980      * the specified threshold and is congested
 981      */
 982     handoff_queue_by_worker_index [vlib_worker_index] = fq;
 983     fq->enqueue_full_events++;
 984     return fq;
 985   }
 986
 987   return NULL;
 988 }
 989
 990 static inline u64 ipv4_get_key (ip4_header_t *ip)
 991 {
 992    u64  hash_key;
 993
 994    hash_key = *((u64*)(&ip->address_pair)) ^ ip->protocol;
 995
 996    return hash_key;
 997 }
 998
 999 static inline u64 ipv6_get_key (ip6_header_t *ip)
1000 {
1001    u64  hash_key;
1002
1003    hash_key = ip->src_address.as_u64[0] ^
1004               rotate_left(ip->src_address.as_u64[1],13) ^
1005               rotate_left(ip->dst_address.as_u64[0],26) ^
1006               rotate_left(ip->dst_address.as_u64[1],39) ^
1007               ip->protocol;
1008
1009    return hash_key;
1010 }
1011
1012
1013 #define MPLS_BOTTOM_OF_STACK_BIT_MASK   0x00000100U
1014 #define MPLS_LABEL_MASK                 0xFFFFF000U
1015
1016 static inline u64 mpls_get_key (mpls_unicast_header_t *m)
1017 {
1018    u64                     hash_key;
1019    u8                      ip_ver;
1020
1021
1022    /* find the bottom of the MPLS label stack. */
1023    if (PREDICT_TRUE(m->label_exp_s_ttl &
1024                     clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK))) {
1025        goto bottom_lbl_found;
1026    }
1027    m++;
1028
1029    if (PREDICT_TRUE(m->label_exp_s_ttl &
1030                     clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK))) {
1031        goto bottom_lbl_found;
1032    }
1033    m++;
1034
1035    if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
1036        goto bottom_lbl_found;
1037    }
1038    m++;
1039
1040    if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
1041        goto bottom_lbl_found;
1042    }
1043    m++;
1044
1045    if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
1046        goto bottom_lbl_found;
1047    }
1048
1049    /* the bottom label was not found - use the last label */
1050    hash_key = m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_LABEL_MASK);
1051
1052    return hash_key;
1053
1054
1055 bottom_lbl_found:
1056    m++;
1057    ip_ver = (*((u8 *)m) >> 4);
1058
1059    /* find out if it is IPV4 or IPV6 header */
1060    if (PREDICT_TRUE(ip_ver == 4)) {
1061        hash_key = ipv4_get_key((ip4_header_t *)m);
1062    } else if (PREDICT_TRUE(ip_ver == 6)) {
1063        hash_key = ipv6_get_key((ip6_header_t *)m);
1064    } else {
1065        /* use the bottom label */
1066        hash_key = (m-1)->label_exp_s_ttl & clib_net_to_host_u32(MPLS_LABEL_MASK);
1067    }
1068
1069    return hash_key;
1070
1071 }
1072
1073 static inline u64 eth_get_key (ethernet_header_t *h0)
1074 {
1075    u64 hash_key;
1076
1077
1078    if (PREDICT_TRUE(h0->type) == clib_host_to_net_u16(ETHERNET_TYPE_IP4)) {
1079        hash_key = ipv4_get_key((ip4_header_t *)(h0+1));
1080    } else if (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_IP6)) {
1081        hash_key = ipv6_get_key((ip6_header_t *)(h0+1));
1082    } else if (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST)) {
1083        hash_key = mpls_get_key((mpls_unicast_header_t *)(h0+1));
1084    } else if ((h0->type == clib_host_to_net_u16(ETHERNET_TYPE_VLAN)) ||
1085               (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_DOT1AD))) {
1086        ethernet_vlan_header_t * outer = (ethernet_vlan_header_t *)(h0 + 1);
1087
1088        outer = (outer->type == clib_host_to_net_u16(ETHERNET_TYPE_VLAN)) ?
1089                                   outer+1 : outer;
1090        if (PREDICT_TRUE(outer->type) == clib_host_to_net_u16(ETHERNET_TYPE_IP4)) {
1091            hash_key = ipv4_get_key((ip4_header_t *)(outer+1));
1092        } else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) {
1093            hash_key = ipv6_get_key((ip6_header_t *)(outer+1));
1094        } else if (outer->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST)) {
1095            hash_key = mpls_get_key((mpls_unicast_header_t *)(outer+1));
1096        }  else {
1097            hash_key = outer->type;
1098        }
1099    } else {
1100        hash_key  = 0;
1101    }
1102
1103    return hash_key;
1104 }
1105
1106 /*
1107  * This function is used when dedicated IO threads feed the worker threads.
1108  *
1109  * Devices are allocated to this thread based on instances and instance_id.
1110  * If instances==0 then the function automatically determines the number
1111  * of instances of this thread, and allocates devices between them.
1112  * If instances != 0, then instance_id must be in the range 0..instances-1.
1113  * The function allocates devices among the specified number of instances,
1114  * with this thread having the given instance id. This option is used for
1115  * splitting devices among differently named "io"-type threads.
1116  */
1117 void dpdk_io_thread (vlib_worker_thread_t * w,
1118                      u32 instances,
1119                      u32 instance_id,
1120                      char *worker_name,
1121                      dpdk_io_thread_callback_t callback)
1122 {
1123   vlib_main_t * vm = vlib_get_main();
1124   vlib_thread_main_t * tm = vlib_get_thread_main();
1125   vlib_thread_registration_t * tr;
1126   dpdk_main_t * dm = &dpdk_main;
1127   char *io_name = w->registration->name;
1128   dpdk_device_t * xd;
1129   dpdk_device_t ** my_devices = 0;
1130   vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index = 0;
1131   vlib_frame_queue_t ** congested_handoff_queue_by_worker_index = 0;
1132   vlib_frame_queue_elt_t * hf = 0;
1133   int i;
1134   u32 n_left_to_next_worker = 0, * to_next_worker = 0;
1135   u32 next_worker_index = 0;
1136   u32 current_worker_index = ~0;
1137   u32 cpu_index = os_get_cpu_number();
1138   u32 num_workers = 0;
1139   u32 num_devices = 0;
1140   uword * p;
1141   u16 queue_id = 0;
1142   vlib_node_runtime_t * node_trace = 0;
1143   u32 first_worker_index = 0;
1144   u32 buffer_flags_template;
1145
1146   /* Wait until the dpdk init sequence is complete */
1147   while (dm->io_thread_release == 0)
1148     vlib_worker_thread_barrier_check();
1149
1150   clib_time_init (&vm->clib_time);
1151
1152   p = hash_get_mem (tm->thread_registrations_by_name, worker_name);
1153   ASSERT (p);
1154   tr = (vlib_thread_registration_t *) p[0];
1155   if (tr)
1156     {
1157       num_workers = tr->count;
1158       first_worker_index = tr->first_index;
1159     }
1160
1161   /* Allocate devices to this thread */
1162   if (instances == 0)
1163     {
1164       /* auto-assign */
1165       instance_id = w->instance_id;
1166
1167       p = hash_get_mem (tm->thread_registrations_by_name, io_name);
1168       tr = (vlib_thread_registration_t *) p[0];
1169       /* Otherwise, how did we get here */
1170       ASSERT (tr && tr->count);
1171       instances = tr->count;
1172     }
1173   else
1174     {
1175       /* manually assign */
1176       ASSERT (instance_id < instances);
1177     }
1178
1179   vec_validate (handoff_queue_elt_by_worker_index,
1180                 first_worker_index + num_workers - 1);
1181
1182   vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1183                            first_worker_index + num_workers - 1,
1184                            (vlib_frame_queue_t *)(~0));
1185
1186   buffer_flags_template = dm->buffer_flags_template;
1187
1188   /* And handle them... */
1189   while (1)
1190     {
1191       u32 n_buffers;
1192       u32 mb_index;
1193       uword n_rx_bytes = 0;
1194       u32 n_trace, trace_cnt __attribute__((unused));
1195       vlib_buffer_free_list_t * fl;
1196       u32 hash;
1197       u64 hash_key;
1198       u8 efd_discard_burst;
1199
1200       vlib_worker_thread_barrier_check ();
1201
1202       /* Invoke callback if supplied */
1203       if (PREDICT_FALSE(callback != NULL))
1204           callback(vm);
1205
1206       if (PREDICT_FALSE(vec_len(dm->devices) != num_devices))
1207       {
1208         vec_reset_length(my_devices);
1209         vec_foreach (xd, dm->devices)
1210           {
1211             if (((xd - dm->devices) % tr->count) == instance_id)
1212               {
1213                 fprintf(stderr, "i/o thread %d (cpu %d) takes port %d\n",
1214                         instance_id, (int) os_get_cpu_number(), (int) (xd - dm->devices));
1215                 vec_add1 (my_devices, xd);
1216               }
1217           }
1218         num_devices = vec_len(dm->devices);
1219       }
1220
1221       for (i = 0; i < vec_len (my_devices); i++)
1222       {
1223           xd = my_devices[i];
1224
1225           if (!xd->admin_up)
1226             continue;
1227
1228           n_buffers = dpdk_rx_burst(dm, xd, 0 /* queue_id */);
1229
1230           if (n_buffers == 0)
1231             {
1232               /* check if EFD (dpdk) is enabled */
1233               if (PREDICT_FALSE(dm->efd.enabled))
1234                 {
1235                   /* reset a few stats */
1236                   xd->efd_agent.last_poll_time = 0;
1237                   xd->efd_agent.last_burst_sz = 0;
1238                 }
1239               continue;
1240             }
1241
1242           trace_cnt = n_trace = 0;
1243           if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
1244             {
1245               /*
1246                * packet tracing is triggered on the dpdk-input node for
1247                * ease-of-use. Re-fetch the node_runtime for dpdk-input
1248                * in case it has changed.
1249                */
1250               node_trace = vlib_node_get_runtime (vm, dpdk_input_node.index);
1251
1252               vec_reset_length (xd->d_trace_buffers);
1253               trace_cnt = n_trace = vlib_get_trace_count (vm, node_trace);
1254             }
1255
1256           /*
1257            * DAW-FIXME: VMXNET3 device stop/start doesn't work,
1258            * therefore fake the stop in the dpdk driver by
1259            * silently dropping all of the incoming pkts instead of
1260            * stopping the driver / hardware.
1261            */
1262           if (PREDICT_FALSE(xd->admin_up != 1))
1263             {
1264               for (mb_index = 0; mb_index < n_buffers; mb_index++)
1265                 rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]);
1266               continue;
1267             }
1268
1269           /* reset EFD action for the burst */
1270           efd_discard_burst = 0;
1271
1272           /* Check for congestion if EFD (Early-Fast-Discard) is enabled
1273            * in any mode (e.g. dpdk, monitor, or drop_all)
1274            */
1275           if (PREDICT_FALSE(dm->efd.enabled))
1276             {
1277               /* update EFD counters */
1278               dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
1279
1280               if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED))
1281                 {
1282                   /* drop all received packets */
1283                   for (mb_index = 0; mb_index < n_buffers; mb_index++)
1284                     rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]);
1285
1286                   xd->efd_agent.discard_cnt += n_buffers;
1287                   increment_efd_drop_counter(vm,
1288                                              DPDK_ERROR_VLAN_EFD_DROP_PKTS,
1289                                              n_buffers);
1290
1291                   continue;
1292                 }
1293
1294               if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >=
1295                                 dm->efd.consec_full_frames_hi_thresh))
1296                 {
1297                   u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index,
1298                                                                queue_id);
1299                   if (device_queue_sz >= dm->efd.queue_hi_thresh)
1300                     {
1301                       /* dpdk device queue has reached the critical threshold */
1302                       xd->efd_agent.congestion_cnt++;
1303
1304                       /* apply EFD to packets from the burst */
1305                       efd_discard_burst = 1;
1306                     }
1307                 }
1308             }
1309
1310           fl = vlib_buffer_get_free_list
1311             (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
1312
1313           mb_index = 0;
1314
1315           while (n_buffers > 0)
1316             {
1317               u32 bi0;
1318               u8 next0, error0;
1319               u32 l3_offset0;
1320               vlib_buffer_t * b0, * b_seg, * b_chain = 0;
1321               ethernet_header_t * h0;
1322               u8 nb_seg = 1;
1323               struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index];
1324               struct rte_mbuf *mb_seg = mb->next;
1325
1326               if (PREDICT_TRUE(n_buffers > 1))
1327                 {
1328                   struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
1329                   vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1330                   CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1331                   CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1332                   CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD);
1333                 }
1334
1335               b0 = vlib_buffer_from_rte_mbuf(mb);
1336
1337               /* check whether EFD is looking for packets to discard */
1338               if (PREDICT_FALSE(efd_discard_burst))
1339                 {
1340                   u32 cntr_type;
1341                   if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
1342                     {
1343                       rte_pktmbuf_free(mb);
1344                       xd->efd_agent.discard_cnt++;
1345                       increment_efd_drop_counter(vm,
1346                                                  cntr_type,
1347                                                  1);
1348
1349                       n_buffers--;
1350                       mb_index++;
1351                       continue;
1352                     }
1353                 }
1354
1355               /* Prefetch one next segment if it exists */
1356               if (PREDICT_FALSE(mb->nb_segs > 1))
1357                 {
1358                   struct rte_mbuf *pfmb = mb->next;
1359                   vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1360                   CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1361                   CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1362                   b_chain = b0;
1363                 }
1364
1365               bi0 = vlib_get_buffer_index (vm, b0);
1366               vlib_buffer_init_for_free_list (b0, fl);
1367               b0->clone_count = 0;
1368
1369               dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
1370                                                        &next0, &error0);
1371 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
1372               /*
1373                * Clear overloaded TX offload flags when a DPDK driver
1374                * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
1375                */
1376               if (PREDICT_TRUE(trace_cnt == 0))
1377                 mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
1378               else
1379                 trace_cnt--;
1380 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
1381
1382               if (error0)
1383                   clib_warning ("bi %d error %d", bi0, error0);
1384
1385               b0->error = 0;
1386
1387               l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT ||
1388                              next0 == DPDK_RX_NEXT_IP6_INPUT ||
1389                              next0 == DPDK_RX_NEXT_MPLS_INPUT) ?
1390                             sizeof (ethernet_header_t) : 0);
1391
1392               b0->current_data = l3_offset0;
1393               b0->current_length = mb->data_len - l3_offset0;
1394
1395               b0->flags = buffer_flags_template;
1396
1397               if (VMWARE_LENGTH_BUG_WORKAROUND)
1398                   b0->current_length -= 4;
1399
1400               vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
1401               vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
1402               vnet_buffer(b0)->io_handoff.next_index = next0;
1403               n_rx_bytes += mb->pkt_len;
1404
1405               /* Process subsequent segments of multi-segment packets */
1406               while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
1407                 {
1408                   ASSERT(mb_seg != 0);
1409
1410                   b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
1411                   vlib_buffer_init_for_free_list (b_seg, fl);
1412                   b_seg->clone_count = 0;
1413
1414                   ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1415                   ASSERT(b_seg->current_data == 0);
1416
1417                   /*
1418                    * The driver (e.g. virtio) may not put the packet data at the start
1419                    * of the segment, so don't assume b_seg->current_data == 0 is correct.
1420                    */
1421                   b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data;
1422
1423                   b_seg->current_length = mb_seg->data_len;
1424                   b0->total_length_not_including_first_buffer +=
1425                     mb_seg->data_len;
1426
1427                   b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
1428                   b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
1429
1430                   b_chain = b_seg;
1431                   mb_seg = mb_seg->next;
1432                   nb_seg++;
1433                 }
1434
1435               /*
1436                * Turn this on if you run into
1437                * "bad monkey" contexts, and you want to know exactly
1438                * which nodes they've visited... See main.c...
1439                */
1440               VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0);
1441
1442               if (PREDICT_FALSE (n_trace > mb_index))
1443                 vec_add1 (xd->d_trace_buffers, bi0);
1444
1445               next_worker_index = first_worker_index;
1446
1447               /*
1448                * Force unknown traffic onto worker 0,
1449                * and into ethernet-input. $$$$ add more hashes.
1450                */
1451               h0 = (ethernet_header_t *) b0->data;
1452
1453               /* Compute ingress LB hash */
1454               hash_key = eth_get_key(h0);
1455               hash = (u32)clib_xxhash(hash_key);
1456
1457               if (PREDICT_TRUE (is_pow2(num_workers)))
1458                 next_worker_index += hash & (num_workers - 1);
1459               else
1460                 next_worker_index += hash % num_workers;
1461
1462               /* if EFD is enabled and not already discarding from dpdk,
1463                * check the worker ring/queue for congestion
1464                */
1465               if (PREDICT_FALSE(tm->efd.enabled && !efd_discard_burst))
1466                 {
1467                   vlib_frame_queue_t *fq;
1468
1469                   /* fq will be valid if the ring is congested */
1470                   fq = is_vlib_handoff_queue_congested(
1471                       next_worker_index, tm->efd.queue_hi_thresh,
1472                       congested_handoff_queue_by_worker_index);
1473
1474                   if (PREDICT_FALSE(fq != NULL))
1475                     {
1476                       u32 cntr_type;
1477                       if (PREDICT_TRUE(cntr_type =
1478                                        is_efd_discardable(tm, b0, mb)))
1479                         {
1480                           /* discard the packet */
1481                           fq->enqueue_efd_discards++;
1482                           increment_efd_drop_counter(vm, cntr_type, 1);
1483                           rte_pktmbuf_free(mb);
1484                           n_buffers--;
1485                           mb_index++;
1486                           continue;
1487                         }
1488                     }
1489                 }
1490
1491               if (next_worker_index != current_worker_index)
1492                 {
1493                   if (hf)
1494                     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1495
1496                   hf = dpdk_get_handoff_queue_elt(
1497                            next_worker_index,
1498                            handoff_queue_elt_by_worker_index);
1499
1500                   n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1501                   to_next_worker = &hf->buffer_index[hf->n_vectors];
1502                   current_worker_index = next_worker_index;
1503                 }
1504
1505               /* enqueue to correct worker thread */
1506               to_next_worker[0] = bi0;
1507               to_next_worker++;
1508               n_left_to_next_worker--;
1509
1510               if (n_left_to_next_worker == 0)
1511                 {
1512                   hf->n_vectors = VLIB_FRAME_SIZE;
1513                   vlib_put_handoff_queue_elt(hf);
1514                   current_worker_index = ~0;
1515                   handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1516                   hf = 0;
1517                 }
1518
1519               n_buffers--;
1520               mb_index++;
1521             }
1522
1523           if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0))
1524             {
1525               /* credit the trace to the trace node */
1526               dpdk_rx_trace (dm, node_trace, xd, queue_id, xd->d_trace_buffers,
1527                              vec_len (xd->d_trace_buffers));
1528               vlib_set_trace_count (vm, node_trace, n_trace - vec_len (xd->d_trace_buffers));
1529             }
1530
1531           vlib_increment_combined_counter
1532             (vnet_get_main()->interface_main.combined_sw_if_counters
1533              + VNET_INTERFACE_COUNTER_RX,
1534              cpu_index,
1535              xd->vlib_sw_if_index,
1536              mb_index, n_rx_bytes);
1537
1538           dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index);
1539           dw->aggregate_rx_packets += mb_index;
1540         }
1541
1542       if (hf)
1543         hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1544
1545       /* Ship frames to the worker nodes */
1546       for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1547         {
1548           if (handoff_queue_elt_by_worker_index[i])
1549             {
1550               hf = handoff_queue_elt_by_worker_index[i];
1551               /*
1552                * It works better to let the handoff node
1553                * rate-adapt, always ship the handoff queue element.
1554                */
1555               if (1 || hf->n_vectors == hf->last_n_vectors)
1556                 {
1557                   vlib_put_handoff_queue_elt(hf);
1558                   handoff_queue_elt_by_worker_index[i] = 0;
1559                 }
1560               else
1561                 hf->last_n_vectors = hf->n_vectors;
1562             }
1563           congested_handoff_queue_by_worker_index[i] = (vlib_frame_queue_t *)(~0);
1564         }
1565       hf = 0;
1566       current_worker_index = ~0;
1567
1568       vlib_increment_main_loop_counter (vm);
1569     }
1570 }
1571
1572 /*
1573  * This function is used when the main thread performs IO and feeds the
1574  * worker threads.
1575  */
1576 static uword
1577 dpdk_io_input (vlib_main_t * vm,
1578                vlib_node_runtime_t * node,
1579                vlib_frame_t * f)
1580 {
1581   dpdk_main_t * dm = &dpdk_main;
1582   dpdk_device_t * xd;
1583   vlib_thread_main_t * tm = vlib_get_thread_main();
1584   uword n_rx_packets = 0;
1585   static vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index;
1586   static vlib_frame_queue_t ** congested_handoff_queue_by_worker_index = 0;
1587   vlib_frame_queue_elt_t * hf = 0;
1588   int i;
1589   u32 n_left_to_next_worker = 0, * to_next_worker = 0;
1590   u32 next_worker_index = 0;
1591   u32 current_worker_index = ~0;
1592   u32 cpu_index = os_get_cpu_number();
1593   static int num_workers_set;
1594   static u32 num_workers;
1595   u16 queue_id = 0;
1596   vlib_node_runtime_t * node_trace;
1597   static u32 first_worker_index;
1598   u32 buffer_flags_template;
1599
1600   if (PREDICT_FALSE(num_workers_set == 0))
1601     {
1602       uword * p;
1603       vlib_thread_registration_t * tr;
1604       /* Only the standard vnet worker threads are supported */
1605       p = hash_get_mem (tm->thread_registrations_by_name, "workers");
1606       tr = (vlib_thread_registration_t *) p[0];
1607       if (tr)
1608         {
1609           num_workers = tr->count;
1610           first_worker_index = tr->first_index;
1611         }
1612       num_workers_set = 1;
1613     }
1614
1615   if (PREDICT_FALSE(handoff_queue_elt_by_worker_index == 0))
1616     {
1617       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1618
1619       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1620                                first_worker_index + num_workers - 1,
1621                                (vlib_frame_queue_t *)(~0));
1622     }
1623
1624   /* packet tracing is triggered on the dpdk-input node for ease-of-use */
1625   node_trace = vlib_node_get_runtime (vm, dpdk_input_node.index);
1626
1627   buffer_flags_template = dm->buffer_flags_template;
1628
1629   vec_foreach (xd, dm->devices)
1630     {
1631       u32 n_buffers;
1632       u32 mb_index;
1633       uword n_rx_bytes = 0;
1634       u32 n_trace, trace_cnt __attribute__((unused));
1635       vlib_buffer_free_list_t * fl;
1636       u32 hash;
1637       u64 hash_key;
1638       u8 efd_discard_burst = 0;
1639
1640       if (!xd->admin_up)
1641         continue;
1642
1643       n_buffers = dpdk_rx_burst(dm, xd, queue_id );
1644
1645       if (n_buffers == 0)
1646         {
1647           /* check if EFD (dpdk) is enabled */
1648           if (PREDICT_FALSE(dm->efd.enabled))
1649             {
1650               /* reset a few stats */
1651               xd->efd_agent.last_poll_time = 0;
1652               xd->efd_agent.last_burst_sz = 0;
1653             }
1654           continue;
1655         }
1656
1657       vec_reset_length (xd->d_trace_buffers);
1658       trace_cnt = n_trace = vlib_get_trace_count (vm, node_trace);
1659
1660       /*
1661        * DAW-FIXME: VMXNET3 device stop/start doesn't work,
1662        * therefore fake the stop in the dpdk driver by
1663        * silently dropping all of the incoming pkts instead of
1664        * stopping the driver / hardware.
1665        */
1666       if (PREDICT_FALSE(xd->admin_up != 1))
1667         {
1668           for (mb_index = 0; mb_index < n_buffers; mb_index++)
1669             rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]);
1670           continue;
1671         }
1672
1673       /* Check for congestion if EFD (Early-Fast-Discard) is enabled
1674        * in any mode (e.g. dpdk, monitor, or drop_all)
1675        */
1676       if (PREDICT_FALSE(dm->efd.enabled))
1677         {
1678           /* update EFD counters */
1679           dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
1680
1681           if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED))
1682             {
1683               /* discard all received packets */
1684               for (mb_index = 0; mb_index < n_buffers; mb_index++)
1685                 rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]);
1686
1687               xd->efd_agent.discard_cnt += n_buffers;
1688               increment_efd_drop_counter(vm,
1689                                          DPDK_ERROR_VLAN_EFD_DROP_PKTS,
1690                                          n_buffers);
1691
1692               continue;
1693             }
1694
1695           if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >=
1696                             dm->efd.consec_full_frames_hi_thresh))
1697             {
1698               u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index,
1699                                                            queue_id);
1700               if (device_queue_sz >= dm->efd.queue_hi_thresh)
1701                 {
1702                   /* dpdk device queue has reached the critical threshold */
1703                   xd->efd_agent.congestion_cnt++;
1704
1705                   /* apply EFD to packets from the burst */
1706                   efd_discard_burst = 1;
1707                 }
1708             }
1709         }
1710
1711       fl = vlib_buffer_get_free_list
1712         (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
1713
1714       mb_index = 0;
1715
1716       while (n_buffers > 0)
1717         {
1718           u32 bi0;
1719           u8 next0, error0;
1720           u32 l3_offset0;
1721           vlib_buffer_t * b0, * b_seg, * b_chain = 0;
1722           ethernet_header_t * h0;
1723           u8 nb_seg = 1;
1724           struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index];
1725           struct rte_mbuf *mb_seg = mb->next;
1726
1727           if (PREDICT_TRUE(n_buffers > 1))
1728             {
1729               struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
1730               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1731               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1732               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1733               CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD);
1734             }
1735
1736           b0 = vlib_buffer_from_rte_mbuf(mb);
1737
1738           /* check whether EFD is looking for packets to discard */
1739           if (PREDICT_FALSE(efd_discard_burst))
1740             {
1741               u32 cntr_type;
1742               if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
1743                 {
1744                   rte_pktmbuf_free(mb);
1745                   xd->efd_agent.discard_cnt++;
1746                   increment_efd_drop_counter(vm,
1747                                              cntr_type,
1748                                              1);
1749
1750                   n_buffers--;
1751                   mb_index++;
1752                   continue;
1753                 }
1754             }
1755
1756           /* Prefetch one next segment if it exists */
1757           if (PREDICT_FALSE(mb->nb_segs > 1))
1758             {
1759               struct rte_mbuf *pfmb = mb->next;
1760               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1761               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1762               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1763               b_chain = b0;
1764             }
1765
1766           bi0 = vlib_get_buffer_index (vm, b0);
1767           vlib_buffer_init_for_free_list (b0, fl);
1768           b0->clone_count = 0;
1769
1770           dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
1771                                                    &next0, &error0);
1772 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
1773           /*
1774            * Clear overloaded TX offload flags when a DPDK driver
1775            * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
1776            */
1777           if (PREDICT_TRUE(trace_cnt == 0))
1778             mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
1779           else
1780             trace_cnt--;
1781 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
1782
1783           if (error0)
1784             clib_warning ("bi %d error %d", bi0, error0);
1785
1786           b0->error = 0;
1787
1788           l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT ||
1789                          next0 == DPDK_RX_NEXT_IP6_INPUT ||
1790                          next0 == DPDK_RX_NEXT_MPLS_INPUT) ?
1791                         sizeof (ethernet_header_t) : 0);
1792
1793           b0->current_data = l3_offset0;
1794           b0->current_length = mb->data_len - l3_offset0;
1795
1796           b0->flags = buffer_flags_template;
1797
1798           if (VMWARE_LENGTH_BUG_WORKAROUND)
1799               b0->current_length -= 4;
1800
1801           vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
1802           vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
1803           vnet_buffer(b0)->io_handoff.next_index = next0;
1804           n_rx_bytes += mb->pkt_len;
1805
1806           /* Process subsequent segments of multi-segment packets */
1807           while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
1808             {
1809               ASSERT(mb_seg != 0);
1810
1811               b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
1812               vlib_buffer_init_for_free_list (b_seg, fl);
1813               b_seg->clone_count = 0;
1814
1815               ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1816               ASSERT(b_seg->current_data == 0);
1817
1818               /*
1819                * The driver (e.g. virtio) may not put the packet data at the start
1820                * of the segment, so don't assume b_seg->current_data == 0 is correct.
1821                */
1822               b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data;
1823
1824               b_seg->current_length = mb_seg->data_len;
1825               b0->total_length_not_including_first_buffer +=
1826                 mb_seg->data_len;
1827
1828               b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
1829               b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
1830
1831               b_chain = b_seg;
1832               mb_seg = mb_seg->next;
1833               nb_seg++;
1834             }
1835
1836           /*
1837            * Turn this on if you run into
1838            * "bad monkey" contexts, and you want to know exactly
1839            * which nodes they've visited... See main.c...
1840            */
1841           VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0);
1842
1843           if (PREDICT_FALSE (n_trace > mb_index))
1844             vec_add1 (xd->d_trace_buffers, bi0);
1845
1846           next_worker_index = first_worker_index;
1847
1848           /*
1849            * Force unknown traffic onto worker 0,
1850            * and into ethernet-input. $$$$ add more hashes.
1851            */
1852           h0 = (ethernet_header_t *) b0->data;
1853
1854           /* Compute ingress LB hash */
1855           hash_key = eth_get_key(h0);
1856           hash = (u32)clib_xxhash(hash_key);
1857
1858           if (PREDICT_TRUE (is_pow2(num_workers)))
1859             next_worker_index += hash & (num_workers - 1);
1860           else
1861             next_worker_index += hash % num_workers;
1862
1863           /* if EFD is enabled and not already discarding from dpdk,
1864            * check the worker ring/queue for congestion
1865            */
1866           if (PREDICT_FALSE(tm->efd.enabled && !efd_discard_burst))
1867             {
1868               vlib_frame_queue_t *fq;
1869
1870               /* fq will be valid if the ring is congested */
1871               fq = is_vlib_handoff_queue_congested(
1872                   next_worker_index, tm->efd.queue_hi_thresh,
1873                   congested_handoff_queue_by_worker_index);
1874
1875               if (PREDICT_FALSE(fq != NULL))
1876                 {
1877                   u32 cntr_type;
1878                   if (PREDICT_TRUE(cntr_type =
1879                                    is_efd_discardable(tm, b0, mb)))
1880                     {
1881                       /* discard the packet */
1882                       fq->enqueue_efd_discards++;
1883                       increment_efd_drop_counter(vm, cntr_type, 1);
1884                       rte_pktmbuf_free(mb);
1885                       n_buffers--;
1886                       mb_index++;
1887                       continue;
1888                     }
1889                 }
1890             }
1891
1892           if (next_worker_index != current_worker_index)
1893             {
1894               if (hf)
1895                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1896
1897               hf = dpdk_get_handoff_queue_elt(
1898                      next_worker_index,
1899                      handoff_queue_elt_by_worker_index);
1900
1901               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1902               to_next_worker = &hf->buffer_index[hf->n_vectors];
1903               current_worker_index = next_worker_index;
1904             }
1905
1906           /* enqueue to correct worker thread */
1907           to_next_worker[0] = bi0;
1908           to_next_worker++;
1909           n_left_to_next_worker--;
1910
1911           if (n_left_to_next_worker == 0)
1912             {
1913               hf->n_vectors = VLIB_FRAME_SIZE;
1914               vlib_put_handoff_queue_elt(hf);
1915               current_worker_index = ~0;
1916               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1917               hf = 0;
1918             }
1919
1920           n_buffers--;
1921           mb_index++;
1922         }
1923
1924       if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0))
1925         {
1926           /* credit the trace to the trace node */
1927           dpdk_rx_trace (dm, node_trace, xd, queue_id, xd->d_trace_buffers,
1928                          vec_len (xd->d_trace_buffers));
1929           vlib_set_trace_count (vm, node_trace, n_trace - vec_len (xd->d_trace_buffers));
1930         }
1931
1932       vlib_increment_combined_counter
1933         (vnet_get_main()->interface_main.combined_sw_if_counters
1934          + VNET_INTERFACE_COUNTER_RX,
1935          cpu_index,
1936          xd->vlib_sw_if_index,
1937          mb_index, n_rx_bytes);
1938
1939       dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index);
1940       dw->aggregate_rx_packets += mb_index;
1941       n_rx_packets += mb_index;
1942     }
1943
1944   if (hf)
1945     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1946
1947   /* Ship frames to the worker nodes */
1948   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1949     {
1950       if (handoff_queue_elt_by_worker_index[i])
1951         {
1952           hf = handoff_queue_elt_by_worker_index[i];
1953           /*
1954            * It works better to let the handoff node
1955            * rate-adapt, always ship the handoff queue element.
1956            */
1957           if (1 || hf->n_vectors == hf->last_n_vectors)
1958             {
1959               vlib_put_handoff_queue_elt(hf);
1960               handoff_queue_elt_by_worker_index[i] = 0;
1961             }
1962           else
1963             hf->last_n_vectors = hf->n_vectors;
1964         }
1965       congested_handoff_queue_by_worker_index[i] = (vlib_frame_queue_t *)(~0);
1966     }
1967   hf = 0;
1968   current_worker_index = ~0;
1969   return n_rx_packets;
1970 }
1971
1972 VLIB_REGISTER_NODE (dpdk_io_input_node) = {
1973   .function = dpdk_io_input,
1974   .type = VLIB_NODE_TYPE_INPUT,
1975   .name = "dpdk-io-input",
1976
1977   /* Will be enabled if/when hardware is detected. */
1978   .state = VLIB_NODE_STATE_DISABLED,
1979
1980   .format_buffer = format_ethernet_header_with_length,
1981   .format_trace = format_dpdk_rx_dma_trace,
1982
1983   .n_errors = DPDK_N_ERROR,
1984   .error_strings = dpdk_error_strings,
1985
1986   .n_next_nodes = DPDK_RX_N_NEXT,
1987   .next_nodes = {
1988     [DPDK_RX_NEXT_DROP] = "error-drop",
1989     [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
1990     [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1991     [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
1992     [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
1993   },
1994 };
1995
1996 /*
1997  * set_efd_bitmap()
1998  * Based on the operation type, set lower/upper bits for the given index value
1999  */
2000 void
2001 set_efd_bitmap (u8 *bitmap, u32 value, u32 op)
2002 {
2003     int ix;
2004
2005     *bitmap = 0;
2006     for (ix = 0; ix < 8; ix++) {
2007         if (((op == EFD_OPERATION_LESS_THAN) && (ix < value)) ||
2008             ((op == EFD_OPERATION_GREATER_OR_EQUAL) && (ix >= value))){
2009             (*bitmap) |= (1 << ix);
2010         }
2011     }
2012 }
2013
2014 void
2015 efd_config (u32 enabled,
2016             u32 ip_prec,  u32 ip_op,
2017             u32 mpls_exp, u32 mpls_op,
2018             u32 vlan_cos, u32 vlan_op)
2019 {
2020    vlib_thread_main_t * tm = vlib_get_thread_main();
2021    dpdk_main_t * dm = &dpdk_main;
2022
2023    if (enabled) {
2024        tm->efd.enabled |= VLIB_EFD_DISCARD_ENABLED;
2025        dm->efd.enabled |= DPDK_EFD_DISCARD_ENABLED;
2026    } else {
2027        tm->efd.enabled &= ~VLIB_EFD_DISCARD_ENABLED;
2028        dm->efd.enabled &= ~DPDK_EFD_DISCARD_ENABLED;
2029    }
2030
2031    set_efd_bitmap(&tm->efd.ip_prec_bitmap, ip_prec, ip_op);
2032    set_efd_bitmap(&tm->efd.mpls_exp_bitmap, mpls_exp, mpls_op);
2033    set_efd_bitmap(&tm->efd.vlan_cos_bitmap, vlan_cos, vlan_op);
2034
2035 }