vnet/vnet/devices/dpdk/node.c

   1 /*
   2  * Copyright (c) 2015 Cisco and/or its affiliates.
   3  * Licensed under the Apache License, Version 2.0 (the "License");
   4  * you may not use this file except in compliance with the License.
   5  * You may obtain a copy of the License at:
   6  *
   7  *     http://www.apache.org/licenses/LICENSE-2.0
   8  *
   9  * Unless required by applicable law or agreed to in writing, software
  10  * distributed under the License is distributed on an "AS IS" BASIS,
  11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12  * See the License for the specific language governing permissions and
  13  * limitations under the License.
  14  */
  15 #include <vnet/vnet.h>
  16 #include <vppinfra/vec.h>
  17 #include <vppinfra/error.h>
  18 #include <vppinfra/format.h>
  19 #include <vppinfra/xxhash.h>
  20
  21 #include <vnet/ethernet/ethernet.h>
  22 #include <vnet/devices/dpdk/dpdk.h>
  23 #include <vnet/classify/vnet_classify.h>
  24 #include <vnet/mpls-gre/packet.h>
  25
  26 #include "dpdk_priv.h"
  27
  28 #ifndef MAX
  29 #define MAX(a,b) ((a) < (b) ? (b) : (a))
  30 #endif
  31
  32 #ifndef MIN
  33 #define MIN(a,b) ((a) < (b) ? (a) : (b))
  34 #endif
  35
  36 /*
  37  * At least in certain versions of ESXi, vmware e1000's don't honor the
  38  * "strip rx CRC" bit. Set this flag to work around that bug FOR UNIT TEST ONLY.
  39  *
  40  * If wireshark complains like so:
  41  *
  42  * "Frame check sequence: 0x00000000 [incorrect, should be <hex-num>]"
  43  * and you're using ESXi emulated e1000's, set this flag FOR UNIT TEST ONLY.
  44  *
  45  * Note: do NOT check in this file with this workaround enabled! You'll lose
  46  * actual data from e.g. 10xGE interfaces. The extra 4 bytes annoy
  47  * wireshark, but they're harmless...
  48  */
  49 #define VMWARE_LENGTH_BUG_WORKAROUND 0
  50
  51 typedef struct {
  52   u32 cached_next_index;
  53
  54   /* convenience variables */
  55   vlib_main_t * vlib_main;
  56   vnet_main_t * vnet_main;
  57 } handoff_dispatch_main_t;
  58
  59 typedef struct {
  60   u32 buffer_index;
  61   u32 next_index;
  62   u32 sw_if_index;
  63 } handoff_dispatch_trace_t;
  64
  65 /* packet trace format function */
  66 static u8 * format_handoff_dispatch_trace (u8 * s, va_list * args)
  67 {
  68   CLIB_UNUSED (vlib_main_t * vm) = va_arg (*args, vlib_main_t *);
  69   CLIB_UNUSED (vlib_node_t * node) = va_arg (*args, vlib_node_t *);
  70   handoff_dispatch_trace_t * t = va_arg (*args, handoff_dispatch_trace_t *);
  71
  72   s = format (s, "HANDOFF_DISPATCH: sw_if_index %d next_index %d buffer 0x%x",
  73       t->sw_if_index,
  74       t->next_index,
  75       t->buffer_index);
  76   return s;
  77 }
  78
  79 handoff_dispatch_main_t handoff_dispatch_main;
  80
  81 vlib_node_registration_t handoff_dispatch_node;
  82
  83 #define foreach_handoff_dispatch_error \
  84 _(EXAMPLE, "example packets")
  85
  86 typedef enum {
  87 #define _(sym,str) HANDOFF_DISPATCH_ERROR_##sym,
  88   foreach_handoff_dispatch_error
  89 #undef _
  90   HANDOFF_DISPATCH_N_ERROR,
  91 } handoff_dispatch_error_t;
  92
  93 static char * handoff_dispatch_error_strings[] = {
  94 #define _(sym,string) string,
  95   foreach_handoff_dispatch_error
  96 #undef _
  97 };
  98
  99 static inline
 100 void vlib_put_handoff_queue_elt (vlib_frame_queue_elt_t * hf)
 101 {
 102   CLIB_MEMORY_BARRIER();
 103   hf->valid = 1;
 104 }
 105
 106 static uword
 107 handoff_dispatch_node_fn (vlib_main_t * vm,
 108                   vlib_node_runtime_t * node,
 109                   vlib_frame_t * frame)
 110 {
 111   u32 n_left_from, * from, * to_next;
 112   dpdk_rx_next_t next_index;
 113
 114   from = vlib_frame_vector_args (frame);
 115   n_left_from = frame->n_vectors;
 116   next_index = node->cached_next_index;
 117
 118   while (n_left_from > 0)
 119     {
 120       u32 n_left_to_next;
 121
 122       vlib_get_next_frame (vm, node, next_index,
 123                            to_next, n_left_to_next);
 124
 125       while (n_left_from >= 4 && n_left_to_next >= 2)
 126         {
 127           u32 bi0, bi1;
 128           vlib_buffer_t * b0, * b1;
 129           u32 next0, next1;
 130           u32 sw_if_index0, sw_if_index1;
 131
 132           /* Prefetch next iteration. */
 133           {
 134             vlib_buffer_t * p2, * p3;
 135
 136             p2 = vlib_get_buffer (vm, from[2]);
 137             p3 = vlib_get_buffer (vm, from[3]);
 138
 139             vlib_prefetch_buffer_header (p2, LOAD);
 140             vlib_prefetch_buffer_header (p3, LOAD);
 141           }
 142
 143           /* speculatively enqueue b0 and b1 to the current next frame */
 144           to_next[0] = bi0 = from[0];
 145           to_next[1] = bi1 = from[1];
 146           from += 2;
 147           to_next += 2;
 148           n_left_from -= 2;
 149           n_left_to_next -= 2;
 150
 151           b0 = vlib_get_buffer (vm, bi0);
 152           b1 = vlib_get_buffer (vm, bi1);
 153
 154           next0 = vnet_buffer(b0)->io_handoff.next_index;
 155           next1 = vnet_buffer(b1)->io_handoff.next_index;
 156
 157           if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
 158             {
 159             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 160               {
 161                 vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
 162                 handoff_dispatch_trace_t *t =
 163                   vlib_add_trace (vm, node, b0, sizeof (*t));
 164                 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 165                 t->sw_if_index = sw_if_index0;
 166                 t->next_index = next0;
 167                 t->buffer_index = bi0;
 168               }
 169             if (PREDICT_FALSE(b1->flags & VLIB_BUFFER_IS_TRACED))
 170               {
 171                 vlib_trace_buffer (vm, node, next1, b1, /* follow_chain */ 0);
 172                 handoff_dispatch_trace_t *t =
 173                   vlib_add_trace (vm, node, b1, sizeof (*t));
 174                 sw_if_index1 = vnet_buffer(b1)->sw_if_index[VLIB_RX];
 175                 t->sw_if_index = sw_if_index1;
 176                 t->next_index = next1;
 177                 t->buffer_index = bi1;
 178               }
 179             }
 180
 181           /* verify speculative enqueues, maybe switch current next frame */
 182           vlib_validate_buffer_enqueue_x2 (vm, node, next_index,
 183                                            to_next, n_left_to_next,
 184                                            bi0, bi1, next0, next1);
 185         }
 186
 187       while (n_left_from > 0 && n_left_to_next > 0)
 188         {
 189           u32 bi0;
 190           vlib_buffer_t * b0;
 191           u32 next0;
 192           u32 sw_if_index0;
 193
 194           /* speculatively enqueue b0 to the current next frame */
 195           bi0 = from[0];
 196           to_next[0] = bi0;
 197           from += 1;
 198           to_next += 1;
 199           n_left_from -= 1;
 200           n_left_to_next -= 1;
 201
 202           b0 = vlib_get_buffer (vm, bi0);
 203
 204           next0 = vnet_buffer(b0)->io_handoff.next_index;
 205
 206           if (PREDICT_FALSE(vm->trace_main.trace_active_hint))
 207             {
 208             if (PREDICT_FALSE(b0->flags & VLIB_BUFFER_IS_TRACED))
 209               {
 210                 vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
 211                 handoff_dispatch_trace_t *t =
 212                   vlib_add_trace (vm, node, b0, sizeof (*t));
 213                 sw_if_index0 = vnet_buffer(b0)->sw_if_index[VLIB_RX];
 214                 t->sw_if_index = sw_if_index0;
 215                 t->next_index = next0;
 216                 t->buffer_index = bi0;
 217               }
 218             }
 219
 220           /* verify speculative enqueue, maybe switch current next frame */
 221           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 222                                            to_next, n_left_to_next,
 223                                            bi0, next0);
 224         }
 225
 226       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 227     }
 228
 229   return frame->n_vectors;
 230 }
 231
 232 VLIB_REGISTER_NODE (handoff_dispatch_node) = {
 233   .function = handoff_dispatch_node_fn,
 234   .name = "handoff-dispatch",
 235   .vector_size = sizeof (u32),
 236   .format_trace = format_handoff_dispatch_trace,
 237   .type = VLIB_NODE_TYPE_INTERNAL,
 238   .flags = VLIB_NODE_FLAG_IS_HANDOFF,
 239
 240   .n_errors = ARRAY_LEN(handoff_dispatch_error_strings),
 241   .error_strings = handoff_dispatch_error_strings,
 242
 243   .n_next_nodes = DPDK_RX_N_NEXT,
 244
 245   .next_nodes = {
 246         [DPDK_RX_NEXT_DROP] = "error-drop",
 247         [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
 248         [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input",
 249         [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
 250         [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
 251   },
 252 };
 253
 254 clib_error_t *handoff_dispatch_init (vlib_main_t *vm)
 255 {
 256   handoff_dispatch_main_t * mp = &handoff_dispatch_main;
 257
 258   mp->vlib_main = vm;
 259   mp->vnet_main = &vnet_main;
 260
 261   return 0;
 262 }
 263
 264 VLIB_INIT_FUNCTION (handoff_dispatch_init);
 265
 266 u32 dpdk_get_handoff_node_index (void)
 267 {
 268   return handoff_dispatch_node.index;
 269 }
 270
 271 static char * dpdk_error_strings[] = {
 272 #define _(n,s) s,
 273     foreach_dpdk_error
 274 #undef _
 275 };
 276
 277 always_inline void
 278 dpdk_rx_next_and_error_from_mb_flags_x1 (dpdk_device_t *xd, struct rte_mbuf *mb,
 279                                          vlib_buffer_t *b0,
 280                                          u8 * next0, u8 * error0)
 281 {
 282   u8 is0_ip4, is0_ip6, is0_mpls, n0;
 283   uint16_t mb_flags = mb->ol_flags;
 284
 285   if (PREDICT_FALSE(mb_flags & (
 286 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 287        PKT_EXT_RX_PKT_ERROR | PKT_EXT_RX_BAD_FCS   |
 288 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 289         PKT_RX_IP_CKSUM_BAD  | PKT_RX_L4_CKSUM_BAD
 290     )))
 291     {
 292       /* some error was flagged. determine the drop reason */
 293       n0 = DPDK_RX_NEXT_DROP;
 294       *error0 =
 295 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 296         (mb_flags & PKT_EXT_RX_PKT_ERROR) ? DPDK_ERROR_RX_PACKET_ERROR :
 297         (mb_flags & PKT_EXT_RX_BAD_FCS) ? DPDK_ERROR_RX_BAD_FCS :
 298 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 299         (mb_flags & PKT_RX_IP_CKSUM_BAD) ? DPDK_ERROR_IP_CHECKSUM_ERROR :
 300         (mb_flags & PKT_RX_L4_CKSUM_BAD) ? DPDK_ERROR_L4_CHECKSUM_ERROR :
 301         DPDK_ERROR_NONE;
 302     }
 303   else
 304     {
 305       *error0 = DPDK_ERROR_NONE;
 306       if (xd->per_interface_next_index != ~0)
 307         n0 = xd->per_interface_next_index;
 308       else if (mb_flags & PKT_RX_VLAN_PKT)
 309         n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
 310       else
 311         {
 312           n0 = DPDK_RX_NEXT_ETHERNET_INPUT;
 313 #if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
 314           is0_ip4 = RTE_ETH_IS_IPV4_HDR(mb->packet_type) != 0;
 315 #else
 316           is0_ip4 = (mb_flags & (PKT_RX_IPV4_HDR | PKT_RX_IPV4_HDR_EXT)) != 0;
 317 #endif
 318
 319           if (PREDICT_TRUE(is0_ip4))
 320             n0 = DPDK_RX_NEXT_IP4_INPUT;
 321           else
 322             {
 323 #if RTE_VERSION >= RTE_VERSION_NUM(2, 1, 0, 0)
 324               is0_ip6 = RTE_ETH_IS_IPV6_HDR(mb->packet_type) != 0;
 325 #else
 326               is0_ip6 =
 327                       (mb_flags & (PKT_RX_IPV6_HDR | PKT_RX_IPV6_HDR_EXT)) != 0;
 328 #endif
 329               if (PREDICT_TRUE(is0_ip6))
 330                 n0 = DPDK_RX_NEXT_IP6_INPUT;
 331               else
 332                 {
 333                   ethernet_header_t *h0 = (ethernet_header_t *) b0->data;
 334                   is0_mpls = (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST));
 335                   n0 = is0_mpls ? DPDK_RX_NEXT_MPLS_INPUT : n0;
 336                 }
 337             }
 338         }
 339     }
 340   *next0 = n0;
 341 }
 342
 343 void dpdk_rx_trace (dpdk_main_t * dm,
 344                     vlib_node_runtime_t * node,
 345                     dpdk_device_t * xd,
 346                     u16 queue_id,
 347                     u32 * buffers,
 348                     uword n_buffers)
 349 {
 350   vlib_main_t * vm = vlib_get_main();
 351   u32 * b, n_left;
 352   u8 next0;
 353
 354   n_left = n_buffers;
 355   b = buffers;
 356
 357   while (n_left >= 1)
 358     {
 359       u32 bi0;
 360       vlib_buffer_t * b0;
 361       dpdk_rx_dma_trace_t * t0;
 362       struct rte_mbuf *mb;
 363       u8 error0;
 364
 365       bi0 = b[0];
 366       n_left -= 1;
 367
 368       b0 = vlib_get_buffer (vm, bi0);
 369       mb = rte_mbuf_from_vlib_buffer(b0);
 370       dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
 371                                                &next0, &error0);
 372       vlib_trace_buffer (vm, node, next0, b0, /* follow_chain */ 0);
 373       t0 = vlib_add_trace (vm, node, b0, sizeof (t0[0]));
 374       t0->queue_index = queue_id;
 375       t0->device_index = xd->device_index;
 376       t0->buffer_index = bi0;
 377
 378       memcpy (&t0->mb, mb, sizeof (t0->mb));
 379       memcpy (&t0->buffer, b0, sizeof (b0[0]) - sizeof (b0->pre_data));
 380       memcpy (t0->buffer.pre_data, b0->data, sizeof (t0->buffer.pre_data));
 381
 382 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 383       /*
 384        * Clear overloaded TX offload flags when a DPDK driver
 385        * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
 386        */
 387       mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
 388 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 389
 390       b += 1;
 391     }
 392 }
 393
 394 /*
 395  * dpdk_efd_update_counters()
 396  * Update EFD (early-fast-discard) counters
 397  */
 398 void dpdk_efd_update_counters (dpdk_device_t *xd,
 399                                u32 n_buffers,
 400                                u16 enabled)
 401 {
 402   if (enabled & DPDK_EFD_MONITOR_ENABLED)
 403     {
 404       u64 now = clib_cpu_time_now();
 405       if (xd->efd_agent.last_poll_time > 0)
 406         {
 407           u64 elapsed_time = (now - xd->efd_agent.last_poll_time);
 408           if (elapsed_time > xd->efd_agent.max_poll_delay)
 409             xd->efd_agent.max_poll_delay = elapsed_time;
 410         }
 411       xd->efd_agent.last_poll_time = now;
 412     }
 413
 414   xd->efd_agent.total_packet_cnt += n_buffers;
 415   xd->efd_agent.last_burst_sz = n_buffers;
 416
 417   if (n_buffers > xd->efd_agent.max_burst_sz)
 418     xd->efd_agent.max_burst_sz = n_buffers;
 419
 420   if (PREDICT_FALSE(n_buffers == VLIB_FRAME_SIZE))
 421     {
 422       xd->efd_agent.full_frames_cnt++;
 423       xd->efd_agent.consec_full_frames_cnt++;
 424     }
 425   else
 426     {
 427       xd->efd_agent.consec_full_frames_cnt = 0;
 428     }
 429 }
 430
 431 /* is_efd_discardable()
 432  *   returns non zero DPDK error if packet meets early-fast-discard criteria,
 433  *           zero otherwise
 434  */
 435 u32 is_efd_discardable (vlib_thread_main_t *tm,
 436                         vlib_buffer_t * b0,
 437                         struct rte_mbuf *mb)
 438 {
 439   ethernet_header_t *eh = (ethernet_header_t *) b0->data;
 440
 441   if (eh->type == clib_host_to_net_u16(ETHERNET_TYPE_IP4))
 442     {
 443       ip4_header_t *ipv4 =
 444           (ip4_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 445       u8 pkt_prec = (ipv4->tos >> 5);
 446
 447       return (tm->efd.ip_prec_bitmap & (1 << pkt_prec) ?
 448                   DPDK_ERROR_IPV4_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 449     }
 450   else if (eh->type == clib_net_to_host_u16(ETHERNET_TYPE_IP6))
 451     {
 452       ip6_header_t *ipv6 =
 453           (ip6_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 454       u8 pkt_tclass =
 455           ((ipv6->ip_version_traffic_class_and_flow_label >> 20) & 0xff);
 456
 457       return (tm->efd.ip_prec_bitmap & (1 << pkt_tclass) ?
 458                   DPDK_ERROR_IPV6_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 459     }
 460   else if (eh->type == clib_net_to_host_u16(ETHERNET_TYPE_MPLS_UNICAST))
 461     {
 462       mpls_unicast_header_t *mpls =
 463           (mpls_unicast_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 464       u8 pkt_exp = ((mpls->label_exp_s_ttl >> 9) & 0x07);
 465
 466       return (tm->efd.mpls_exp_bitmap & (1 << pkt_exp) ?
 467                   DPDK_ERROR_MPLS_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 468     }
 469   else if ((eh->type == clib_net_to_host_u16(ETHERNET_TYPE_VLAN)) ||
 470            (eh->type == clib_net_to_host_u16(ETHERNET_TYPE_DOT1AD)))
 471     {
 472       ethernet_vlan_header_t *vlan =
 473           (ethernet_vlan_header_t *)&(b0->data[sizeof(ethernet_header_t)]);
 474       u8 pkt_cos = ((vlan->priority_cfi_and_id >> 13) & 0x07);
 475
 476       return (tm->efd.vlan_cos_bitmap & (1 << pkt_cos) ?
 477                   DPDK_ERROR_VLAN_EFD_DROP_PKTS : DPDK_ERROR_NONE);
 478     }
 479
 480   return DPDK_ERROR_NONE;
 481 }
 482
 483 /*
 484  * This function is used when there are no worker threads.
 485  * The main thread performs IO and forwards the packets.
 486  */
 487 static inline u32 dpdk_device_input ( dpdk_main_t * dm,
 488                                       dpdk_device_t * xd,
 489                                       vlib_node_runtime_t * node,
 490                                       u32 cpu_index,
 491                                       u16 queue_id)
 492 {
 493   u32 n_buffers;
 494   u32 next_index = DPDK_RX_NEXT_ETHERNET_INPUT;
 495   u32 n_left_to_next, * to_next;
 496   u32 mb_index;
 497   vlib_main_t * vm = vlib_get_main();
 498   uword n_rx_bytes = 0;
 499   u32 n_trace, trace_cnt __attribute__((unused));
 500   vlib_buffer_free_list_t * fl;
 501   u8 efd_discard_burst = 0;
 502   u16 ip_align_offset = 0;
 503   u32 buffer_flags_template;
 504
 505   if (xd->admin_up == 0)
 506     return 0;
 507
 508   n_buffers = dpdk_rx_burst(dm, xd, queue_id);
 509
 510   if (n_buffers == 0)
 511     {
 512       /* check if EFD (dpdk) is enabled */
 513       if (PREDICT_FALSE(dm->efd.enabled))
 514         {
 515           /* reset a few stats */
 516           xd->efd_agent.last_poll_time = 0;
 517           xd->efd_agent.last_burst_sz = 0;
 518         }
 519       return 0;
 520     }
 521
 522   if (xd->pmd == VNET_DPDK_PMD_THUNDERX)
 523       ip_align_offset = 6;
 524
 525   buffer_flags_template = dm->buffer_flags_template;
 526
 527   vec_reset_length (xd->d_trace_buffers);
 528   trace_cnt = n_trace = vlib_get_trace_count (vm, node);
 529
 530   fl = vlib_buffer_get_free_list (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
 531
 532   /*
 533    * DAW-FIXME: VMXNET3 device stop/start doesn't work,
 534    * therefore fake the stop in the dpdk driver by
 535    * silently dropping all of the incoming pkts instead of
 536    * stopping the driver / hardware.
 537    */
 538   if (PREDICT_FALSE(xd->admin_up != 1))
 539     {
 540       for (mb_index = 0; mb_index < n_buffers; mb_index++)
 541         rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]);
 542
 543       return 0;
 544     }
 545
 546   /* Check for congestion if EFD (Early-Fast-Discard) is enabled
 547    * in any mode (e.g. dpdk, monitor, or drop_all)
 548    */
 549   if (PREDICT_FALSE(dm->efd.enabled))
 550     {
 551       /* update EFD counters */
 552       dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
 553
 554       if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED))
 555         {
 556           /* discard all received packets */
 557           for (mb_index = 0; mb_index < n_buffers; mb_index++)
 558             rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]);
 559
 560           xd->efd_agent.discard_cnt += n_buffers;
 561           increment_efd_drop_counter(vm,
 562                                      DPDK_ERROR_VLAN_EFD_DROP_PKTS,
 563                                      n_buffers);
 564
 565           return 0;
 566         }
 567
 568       if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >=
 569                         dm->efd.consec_full_frames_hi_thresh))
 570         {
 571           u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index,
 572                                                        queue_id);
 573           if (device_queue_sz >= dm->efd.queue_hi_thresh)
 574             {
 575               /* dpdk device queue has reached the critical threshold */
 576               xd->efd_agent.congestion_cnt++;
 577
 578               /* apply EFD to packets from the burst */
 579               efd_discard_burst = 1;
 580             }
 581         }
 582     }
 583
 584   mb_index = 0;
 585
 586   while (n_buffers > 0)
 587     {
 588       u32 bi0;
 589       u8 next0, error0;
 590       u32 l3_offset0;
 591       vlib_buffer_t * b0, * b_seg, * b_chain = 0;
 592       u32 cntr_type;
 593
 594       vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
 595
 596       while (n_buffers > 0 && n_left_to_next > 0)
 597         {
 598           u8 nb_seg = 1;
 599           struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index];
 600           struct rte_mbuf *mb_seg = mb->next;
 601
 602           if (PREDICT_TRUE(n_buffers > 2))
 603           {
 604               struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
 605               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
 606               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, STORE);
 607               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
 608           }
 609
 610           ASSERT(mb);
 611
 612           b0 = vlib_buffer_from_rte_mbuf(mb);
 613
 614           /* check whether EFD is looking for packets to discard */
 615           if (PREDICT_FALSE(efd_discard_burst))
 616             {
 617               vlib_thread_main_t * tm = vlib_get_thread_main();
 618
 619               if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
 620                 {
 621                   rte_pktmbuf_free(mb);
 622                   xd->efd_agent.discard_cnt++;
 623                   increment_efd_drop_counter(vm,
 624                                              cntr_type,
 625                                              1);
 626                   n_buffers--;
 627                   mb_index++;
 628                   continue;
 629                 }
 630             }
 631
 632           /* Prefetch one next segment if it exists. */
 633           if (PREDICT_FALSE(mb->nb_segs > 1))
 634             {
 635               struct rte_mbuf *pfmb = mb->next;
 636               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
 637               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
 638               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
 639               b_chain = b0;
 640             }
 641
 642           vlib_buffer_init_for_free_list (b0, fl);
 643           b0->clone_count = 0;
 644
 645           bi0 = vlib_get_buffer_index (vm, b0);
 646
 647           to_next[0] = bi0;
 648           to_next++;
 649           n_left_to_next--;
 650
 651           dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
 652                                                    &next0, &error0);
 653 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
 654           /*
 655            * Clear overloaded TX offload flags when a DPDK driver
 656            * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
 657            */
 658
 659           if (PREDICT_TRUE(trace_cnt == 0))
 660             mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
 661           else
 662             trace_cnt--;
 663 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
 664
 665           b0->error = node->errors[error0];
 666
 667           l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT ||
 668                          next0 == DPDK_RX_NEXT_IP6_INPUT ||
 669                          next0 == DPDK_RX_NEXT_MPLS_INPUT) ?
 670                         sizeof (ethernet_header_t) : 0);
 671
 672           b0->current_data = l3_offset0;
 673           b0->current_length = mb->data_len - l3_offset0;
 674
 675           if (PREDICT_FALSE (ip_align_offset != 0))
 676             {
 677               if (next0 == DPDK_RX_NEXT_IP4_INPUT ||
 678                   next0 == DPDK_RX_NEXT_IP6_INPUT)
 679                 b0->current_data += ip_align_offset;
 680             }
 681
 682           b0->flags = buffer_flags_template;
 683
 684           if (VMWARE_LENGTH_BUG_WORKAROUND)
 685               b0->current_length -= 4;
 686
 687           vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
 688           vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
 689           n_rx_bytes += mb->pkt_len;
 690
 691           /* Process subsequent segments of multi-segment packets */
 692           while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
 693             {
 694               ASSERT(mb_seg != 0);
 695
 696               b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
 697               vlib_buffer_init_for_free_list (b_seg, fl);
 698               b_seg->clone_count = 0;
 699
 700               ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
 701               ASSERT(b_seg->current_data == 0);
 702
 703               /*
 704                * The driver (e.g. virtio) may not put the packet data at the start
 705                * of the segment, so don't assume b_seg->current_data == 0 is correct.
 706                */
 707               b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data;
 708
 709               b_seg->current_length = mb_seg->data_len;
 710               b0->total_length_not_including_first_buffer +=
 711                 mb_seg->data_len;
 712
 713               b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
 714               b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
 715
 716               b_chain = b_seg;
 717               mb_seg = mb_seg->next;
 718               nb_seg++;
 719             }
 720
 721           /*
 722            * Turn this on if you run into
 723            * "bad monkey" contexts, and you want to know exactly
 724            * which nodes they've visited... See main.c...
 725            */
 726           VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0);
 727
 728           vlib_validate_buffer_enqueue_x1 (vm, node, next_index,
 729                                            to_next, n_left_to_next,
 730                                            bi0, next0);
 731           if (PREDICT_FALSE (n_trace > mb_index))
 732             vec_add1 (xd->d_trace_buffers, bi0);
 733           n_buffers--;
 734           mb_index++;
 735         }
 736       vlib_put_next_frame (vm, node, next_index, n_left_to_next);
 737     }
 738
 739   if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0))
 740     {
 741       dpdk_rx_trace (dm, node, xd, queue_id, xd->d_trace_buffers,
 742                      vec_len (xd->d_trace_buffers));
 743       vlib_set_trace_count (vm, node, n_trace - vec_len (xd->d_trace_buffers));
 744     }
 745
 746   vlib_increment_combined_counter
 747     (vnet_get_main()->interface_main.combined_sw_if_counters
 748      + VNET_INTERFACE_COUNTER_RX,
 749      cpu_index,
 750      xd->vlib_sw_if_index,
 751      mb_index, n_rx_bytes);
 752
 753   dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index);
 754   dw->aggregate_rx_packets += mb_index;
 755
 756   return mb_index;
 757 }
 758
 759 #if VIRL > 0
 760 #define VIRL_SPEED_LIMIT()                         \
 761   /* Limit the input rate to 1000 vectors / sec */ \
 762   {                                                \
 763     struct timespec ts, tsrem;                     \
 764                                                    \
 765     ts.tv_sec = 0;                                 \
 766     ts.tv_nsec = 1000*1000; /* 1ms */              \
 767                                                    \
 768     while (nanosleep(&ts, &tsrem) < 0)             \
 769       {                                            \
 770         ts = tsrem;                                \
 771       }                                            \
 772   }
 773 #else
 774 #define VIRL_SPEED_LIMIT()
 775 #endif
 776
 777
 778 static uword
 779 dpdk_input (vlib_main_t * vm,
 780             vlib_node_runtime_t * node,
 781             vlib_frame_t * f)
 782 {
 783   dpdk_main_t * dm = &dpdk_main;
 784   dpdk_device_t * xd;
 785   uword n_rx_packets = 0;
 786   dpdk_device_and_queue_t * dq;
 787   u32 cpu_index = os_get_cpu_number();
 788
 789   /*
 790    * Poll all devices on this cpu for input/interrupts.
 791    */
 792   vec_foreach (dq, dm->devices_by_cpu[cpu_index])
 793     {
 794       xd = vec_elt_at_index(dm->devices, dq->device);
 795       ASSERT(dq->queue_id == 0);
 796       n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, 0);
 797     }
 798
 799   VIRL_SPEED_LIMIT()
 800
 801   return n_rx_packets;
 802 }
 803
 804 uword
 805 dpdk_input_rss (vlib_main_t * vm,
 806       vlib_node_runtime_t * node,
 807       vlib_frame_t * f)
 808 {
 809   dpdk_main_t * dm = &dpdk_main;
 810   dpdk_device_t * xd;
 811   uword n_rx_packets = 0;
 812   dpdk_device_and_queue_t * dq;
 813   u32 cpu_index = os_get_cpu_number();
 814
 815   /*
 816    * Poll all devices on this cpu for input/interrupts.
 817    */
 818   vec_foreach (dq, dm->devices_by_cpu[cpu_index])
 819     {
 820       xd = vec_elt_at_index(dm->devices, dq->device);
 821       n_rx_packets += dpdk_device_input (dm, xd, node, cpu_index, dq->queue_id);
 822     }
 823
 824   VIRL_SPEED_LIMIT()
 825
 826   return n_rx_packets;
 827 }
 828
 829 VLIB_REGISTER_NODE (dpdk_input_node) = {
 830   .function = dpdk_input,
 831   .type = VLIB_NODE_TYPE_INPUT,
 832   .name = "dpdk-input",
 833
 834   /* Will be enabled if/when hardware is detected. */
 835   .state = VLIB_NODE_STATE_DISABLED,
 836
 837   .format_buffer = format_ethernet_header_with_length,
 838   .format_trace = format_dpdk_rx_dma_trace,
 839
 840   .n_errors = DPDK_N_ERROR,
 841   .error_strings = dpdk_error_strings,
 842
 843   .n_next_nodes = DPDK_RX_N_NEXT,
 844   .next_nodes = {
 845     [DPDK_RX_NEXT_DROP] = "error-drop",
 846     [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
 847     [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
 848     [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
 849     [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
 850   },
 851 };
 852
 853 /*
 854  * Override the next nodes for the dpdk input nodes.
 855  * Must be invoked prior to VLIB_INIT_FUNCTION calls.
 856  */
 857 void dpdk_set_next_node (dpdk_rx_next_t next, char *name)
 858 {
 859   vlib_node_registration_t *r = &dpdk_input_node;
 860   vlib_node_registration_t *r_io = &dpdk_io_input_node;
 861   vlib_node_registration_t *r_handoff = &handoff_dispatch_node;
 862
 863   switch (next)
 864     {
 865     case DPDK_RX_NEXT_IP4_INPUT:
 866     case DPDK_RX_NEXT_IP6_INPUT:
 867     case DPDK_RX_NEXT_MPLS_INPUT:
 868     case DPDK_RX_NEXT_ETHERNET_INPUT:
 869       r->next_nodes[next] = name;
 870       r_io->next_nodes[next] = name;
 871       r_handoff->next_nodes[next] = name;
 872       break;
 873
 874     default:
 875       clib_warning ("%s: illegal next %d\n", __FUNCTION__, next);
 876       break;
 877     }
 878 }
 879
 880 inline vlib_frame_queue_elt_t *
 881 vlib_get_handoff_queue_elt (u32 vlib_worker_index)
 882 {
 883   vlib_frame_queue_t *fq;
 884   vlib_frame_queue_elt_t *elt;
 885   u64 new_tail;
 886
 887   fq = vlib_frame_queues[vlib_worker_index];
 888   ASSERT (fq);
 889
 890   new_tail = __sync_add_and_fetch (&fq->tail, 1);
 891
 892   /* Wait until a ring slot is available */
 893   while (new_tail >= fq->head_hint + fq->nelts)
 894       vlib_worker_thread_barrier_check ();
 895
 896   elt = fq->elts + (new_tail & (fq->nelts-1));
 897
 898   /* this would be very bad... */
 899   while (elt->valid)
 900     ;
 901
 902   elt->msg_type = VLIB_FRAME_QUEUE_ELT_DISPATCH_FRAME;
 903   elt->last_n_vectors = elt->n_vectors = 0;
 904
 905   return elt;
 906 }
 907
 908 static inline vlib_frame_queue_elt_t *
 909 dpdk_get_handoff_queue_elt (
 910     u32 vlib_worker_index,
 911     vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index)
 912 {
 913   vlib_frame_queue_elt_t *elt;
 914
 915   if (handoff_queue_elt_by_worker_index [vlib_worker_index])
 916       return handoff_queue_elt_by_worker_index [vlib_worker_index];
 917
 918   elt = vlib_get_handoff_queue_elt (vlib_worker_index);
 919
 920   handoff_queue_elt_by_worker_index [vlib_worker_index] = elt;
 921
 922   return elt;
 923 }
 924
 925 static inline vlib_frame_queue_t *
 926 is_vlib_handoff_queue_congested (
 927     u32 vlib_worker_index,
 928     u32 queue_hi_thresh,
 929     vlib_frame_queue_t ** handoff_queue_by_worker_index)
 930 {
 931   vlib_frame_queue_t *fq;
 932
 933   fq = handoff_queue_by_worker_index [vlib_worker_index];
 934   if (fq != (vlib_frame_queue_t *)(~0))
 935       return fq;
 936
 937   fq = vlib_frame_queues[vlib_worker_index];
 938   ASSERT (fq);
 939
 940   if (PREDICT_FALSE(fq->tail >= (fq->head_hint + queue_hi_thresh))) {
 941     /* a valid entry in the array will indicate the queue has reached
 942      * the specified threshold and is congested
 943      */
 944     handoff_queue_by_worker_index [vlib_worker_index] = fq;
 945     fq->enqueue_full_events++;
 946     return fq;
 947   }
 948
 949   return NULL;
 950 }
 951
 952 static inline u64 ipv4_get_key (ip4_header_t *ip)
 953 {
 954    u64  hash_key;
 955
 956    hash_key = *((u64*)(&ip->address_pair)) ^ ip->protocol;
 957
 958    return hash_key;
 959 }
 960
 961 static inline u64 ipv6_get_key (ip6_header_t *ip)
 962 {
 963    u64  hash_key;
 964
 965    hash_key = ip->src_address.as_u64[0] ^
 966               rotate_left(ip->src_address.as_u64[1],13) ^
 967               rotate_left(ip->dst_address.as_u64[0],26) ^
 968               rotate_left(ip->dst_address.as_u64[1],39) ^
 969               ip->protocol;
 970
 971    return hash_key;
 972 }
 973
 974
 975 #define MPLS_BOTTOM_OF_STACK_BIT_MASK   0x00000100U
 976 #define MPLS_LABEL_MASK                 0xFFFFF000U
 977
 978 static inline u64 mpls_get_key (mpls_unicast_header_t *m)
 979 {
 980    u64                     hash_key;
 981    u8                      ip_ver;
 982
 983
 984    /* find the bottom of the MPLS label stack. */
 985    if (PREDICT_TRUE(m->label_exp_s_ttl &
 986                     clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK))) {
 987        goto bottom_lbl_found;
 988    }
 989    m++;
 990
 991    if (PREDICT_TRUE(m->label_exp_s_ttl &
 992                     clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK))) {
 993        goto bottom_lbl_found;
 994    }
 995    m++;
 996
 997    if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
 998        goto bottom_lbl_found;
 999    }
1000    m++;
1001
1002    if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
1003        goto bottom_lbl_found;
1004    }
1005    m++;
1006
1007    if (m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_BOTTOM_OF_STACK_BIT_MASK)) {
1008        goto bottom_lbl_found;
1009    }
1010
1011    /* the bottom label was not found - use the last label */
1012    hash_key = m->label_exp_s_ttl & clib_net_to_host_u32(MPLS_LABEL_MASK);
1013
1014    return hash_key;
1015
1016
1017 bottom_lbl_found:
1018    m++;
1019    ip_ver = (*((u8 *)m) >> 4);
1020
1021    /* find out if it is IPV4 or IPV6 header */
1022    if (PREDICT_TRUE(ip_ver == 4)) {
1023        hash_key = ipv4_get_key((ip4_header_t *)m);
1024    } else if (PREDICT_TRUE(ip_ver == 6)) {
1025        hash_key = ipv6_get_key((ip6_header_t *)m);
1026    } else {
1027        /* use the bottom label */
1028        hash_key = (m-1)->label_exp_s_ttl & clib_net_to_host_u32(MPLS_LABEL_MASK);
1029    }
1030
1031    return hash_key;
1032
1033 }
1034
1035 static inline u64 eth_get_key (ethernet_header_t *h0)
1036 {
1037    u64 hash_key;
1038
1039
1040    if (PREDICT_TRUE(h0->type) == clib_host_to_net_u16(ETHERNET_TYPE_IP4)) {
1041        hash_key = ipv4_get_key((ip4_header_t *)(h0+1));
1042    } else if (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_IP6)) {
1043        hash_key = ipv6_get_key((ip6_header_t *)(h0+1));
1044    } else if (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST)) {
1045        hash_key = mpls_get_key((mpls_unicast_header_t *)(h0+1));
1046    } else if ((h0->type == clib_host_to_net_u16(ETHERNET_TYPE_VLAN)) ||
1047               (h0->type == clib_host_to_net_u16(ETHERNET_TYPE_DOT1AD))) {
1048        ethernet_vlan_header_t * outer = (ethernet_vlan_header_t *)(h0 + 1);
1049
1050        outer = (outer->type == clib_host_to_net_u16(ETHERNET_TYPE_VLAN)) ?
1051                                   outer+1 : outer;
1052        if (PREDICT_TRUE(outer->type) == clib_host_to_net_u16(ETHERNET_TYPE_IP4)) {
1053            hash_key = ipv4_get_key((ip4_header_t *)(outer+1));
1054        } else if (outer->type == clib_host_to_net_u16 (ETHERNET_TYPE_IP6)) {
1055            hash_key = ipv6_get_key((ip6_header_t *)(outer+1));
1056        } else if (outer->type == clib_host_to_net_u16(ETHERNET_TYPE_MPLS_UNICAST)) {
1057            hash_key = mpls_get_key((mpls_unicast_header_t *)(outer+1));
1058        }  else {
1059            hash_key = outer->type;
1060        }
1061    } else {
1062        hash_key  = 0;
1063    }
1064
1065    return hash_key;
1066 }
1067
1068 /*
1069  * This function is used when dedicated IO threads feed the worker threads.
1070  *
1071  * Devices are allocated to this thread based on instances and instance_id.
1072  * If instances==0 then the function automatically determines the number
1073  * of instances of this thread, and allocates devices between them.
1074  * If instances != 0, then instance_id must be in the range 0..instances-1.
1075  * The function allocates devices among the specified number of instances,
1076  * with this thread having the given instance id. This option is used for
1077  * splitting devices among differently named "io"-type threads.
1078  */
1079 void dpdk_io_thread (vlib_worker_thread_t * w,
1080                      u32 instances,
1081                      u32 instance_id,
1082                      char *worker_name,
1083                      dpdk_io_thread_callback_t callback)
1084 {
1085   vlib_main_t * vm = vlib_get_main();
1086   vlib_thread_main_t * tm = vlib_get_thread_main();
1087   vlib_thread_registration_t * tr;
1088   dpdk_main_t * dm = &dpdk_main;
1089   char *io_name = w->registration->name;
1090   dpdk_device_t * xd;
1091   dpdk_device_t ** my_devices = 0;
1092   vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index = 0;
1093   vlib_frame_queue_t ** congested_handoff_queue_by_worker_index = 0;
1094   vlib_frame_queue_elt_t * hf = 0;
1095   int i;
1096   u32 n_left_to_next_worker = 0, * to_next_worker = 0;
1097   u32 next_worker_index = 0;
1098   u32 current_worker_index = ~0;
1099   u32 cpu_index = os_get_cpu_number();
1100   u32 num_workers = 0;
1101   u32 num_devices = 0;
1102   uword * p;
1103   u16 queue_id = 0;
1104   vlib_node_runtime_t * node_trace;
1105   u32 first_worker_index = 0;
1106   u32 buffer_flags_template;
1107
1108   /* Wait until the dpdk init sequence is complete */
1109   while (dm->io_thread_release == 0)
1110     vlib_worker_thread_barrier_check();
1111
1112   clib_time_init (&vm->clib_time);
1113
1114   p = hash_get_mem (tm->thread_registrations_by_name, worker_name);
1115   ASSERT (p);
1116   tr = (vlib_thread_registration_t *) p[0];
1117   if (tr)
1118     {
1119       num_workers = tr->count;
1120       first_worker_index = tr->first_index;
1121     }
1122
1123   /* Allocate devices to this thread */
1124   if (instances == 0)
1125     {
1126       /* auto-assign */
1127       instance_id = w->instance_id;
1128
1129       p = hash_get_mem (tm->thread_registrations_by_name, io_name);
1130       tr = (vlib_thread_registration_t *) p[0];
1131       /* Otherwise, how did we get here */
1132       ASSERT (tr && tr->count);
1133       instances = tr->count;
1134     }
1135   else
1136     {
1137       /* manually assign */
1138       ASSERT (instance_id < instances);
1139     }
1140
1141   vec_validate (handoff_queue_elt_by_worker_index,
1142                 first_worker_index + num_workers - 1);
1143
1144   vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1145                            first_worker_index + num_workers - 1,
1146                            (vlib_frame_queue_t *)(~0));
1147
1148   /* packet tracing is triggered on the dpdk-input node for ease-of-use */
1149   node_trace = vlib_node_get_runtime (vm, dpdk_input_node.index);
1150
1151   buffer_flags_template = dm->buffer_flags_template;
1152
1153   /* And handle them... */
1154   while (1)
1155     {
1156       u32 n_buffers;
1157       u32 mb_index;
1158       uword n_rx_bytes = 0;
1159       u32 n_trace, trace_cnt __attribute__((unused));
1160       vlib_buffer_free_list_t * fl;
1161       u32 hash;
1162       u64 hash_key;
1163       u8 efd_discard_burst;
1164
1165       vlib_worker_thread_barrier_check ();
1166
1167       /* Invoke callback if supplied */
1168       if (PREDICT_FALSE(callback != NULL))
1169           callback(vm);
1170
1171       if (PREDICT_FALSE(vec_len(dm->devices) != num_devices))
1172       {
1173         vec_reset_length(my_devices);
1174         vec_foreach (xd, dm->devices)
1175           {
1176             if (((xd - dm->devices) % tr->count) == instance_id)
1177               {
1178                 fprintf(stderr, "i/o thread %d (cpu %d) takes port %d\n",
1179                         instance_id, (int) os_get_cpu_number(), (int) (xd - dm->devices));
1180                 vec_add1 (my_devices, xd);
1181               }
1182           }
1183         num_devices = vec_len(dm->devices);
1184       }
1185
1186       for (i = 0; i < vec_len (my_devices); i++)
1187       {
1188           xd = my_devices[i];
1189
1190           if (!xd->admin_up)
1191             continue;
1192
1193           n_buffers = dpdk_rx_burst(dm, xd, 0 /* queue_id */);
1194
1195           if (n_buffers == 0)
1196             {
1197               /* check if EFD (dpdk) is enabled */
1198               if (PREDICT_FALSE(dm->efd.enabled))
1199                 {
1200                   /* reset a few stats */
1201                   xd->efd_agent.last_poll_time = 0;
1202                   xd->efd_agent.last_burst_sz = 0;
1203                 }
1204               continue;
1205             }
1206
1207           vec_reset_length (xd->d_trace_buffers);
1208           trace_cnt = n_trace = vlib_get_trace_count (vm, node_trace);
1209
1210           /*
1211            * DAW-FIXME: VMXNET3 device stop/start doesn't work,
1212            * therefore fake the stop in the dpdk driver by
1213            * silently dropping all of the incoming pkts instead of
1214            * stopping the driver / hardware.
1215            */
1216           if (PREDICT_FALSE(xd->admin_up != 1))
1217             {
1218               for (mb_index = 0; mb_index < n_buffers; mb_index++)
1219                 rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]);
1220               continue;
1221             }
1222
1223           /* reset EFD action for the burst */
1224           efd_discard_burst = 0;
1225
1226           /* Check for congestion if EFD (Early-Fast-Discard) is enabled
1227            * in any mode (e.g. dpdk, monitor, or drop_all)
1228            */
1229           if (PREDICT_FALSE(dm->efd.enabled))
1230             {
1231               /* update EFD counters */
1232               dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
1233
1234               if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED))
1235                 {
1236                   /* drop all received packets */
1237                   for (mb_index = 0; mb_index < n_buffers; mb_index++)
1238                     rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]);
1239
1240                   xd->efd_agent.discard_cnt += n_buffers;
1241                   increment_efd_drop_counter(vm,
1242                                              DPDK_ERROR_VLAN_EFD_DROP_PKTS,
1243                                              n_buffers);
1244
1245                   continue;
1246                 }
1247
1248               if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >=
1249                                 dm->efd.consec_full_frames_hi_thresh))
1250                 {
1251                   u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index,
1252                                                                queue_id);
1253                   if (device_queue_sz >= dm->efd.queue_hi_thresh)
1254                     {
1255                       /* dpdk device queue has reached the critical threshold */
1256                       xd->efd_agent.congestion_cnt++;
1257
1258                       /* apply EFD to packets from the burst */
1259                       efd_discard_burst = 1;
1260                     }
1261                 }
1262             }
1263
1264           fl = vlib_buffer_get_free_list
1265             (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
1266
1267           mb_index = 0;
1268
1269           while (n_buffers > 0)
1270             {
1271               u32 bi0;
1272               u8 next0, error0;
1273               u32 l3_offset0;
1274               vlib_buffer_t * b0, * b_seg, * b_chain = 0;
1275               ethernet_header_t * h0;
1276               u8 nb_seg = 1;
1277               struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index];
1278               struct rte_mbuf *mb_seg = mb->next;
1279
1280               if (PREDICT_TRUE(n_buffers > 1))
1281                 {
1282                   struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
1283                   vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1284                   CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1285                   CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1286                   CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD);
1287                 }
1288
1289               b0 = vlib_buffer_from_rte_mbuf(mb);
1290
1291               /* check whether EFD is looking for packets to discard */
1292               if (PREDICT_FALSE(efd_discard_burst))
1293                 {
1294                   u32 cntr_type;
1295                   if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
1296                     {
1297                       rte_pktmbuf_free(mb);
1298                       xd->efd_agent.discard_cnt++;
1299                       increment_efd_drop_counter(vm,
1300                                                  cntr_type,
1301                                                  1);
1302
1303                       n_buffers--;
1304                       mb_index++;
1305                       continue;
1306                     }
1307                 }
1308
1309               /* Prefetch one next segment if it exists */
1310               if (PREDICT_FALSE(mb->nb_segs > 1))
1311                 {
1312                   struct rte_mbuf *pfmb = mb->next;
1313                   vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1314                   CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1315                   CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1316                   b_chain = b0;
1317                 }
1318
1319               bi0 = vlib_get_buffer_index (vm, b0);
1320               vlib_buffer_init_for_free_list (b0, fl);
1321               b0->clone_count = 0;
1322
1323               dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
1324                                                        &next0, &error0);
1325 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
1326               /*
1327                * Clear overloaded TX offload flags when a DPDK driver
1328                * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
1329                */
1330               if (PREDICT_TRUE(trace_cnt == 0))
1331                 mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
1332               else
1333                 trace_cnt--;
1334 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
1335
1336               if (error0)
1337                   clib_warning ("bi %d error %d", bi0, error0);
1338
1339               b0->error = 0;
1340
1341               l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT ||
1342                              next0 == DPDK_RX_NEXT_IP6_INPUT ||
1343                              next0 == DPDK_RX_NEXT_MPLS_INPUT) ?
1344                             sizeof (ethernet_header_t) : 0);
1345
1346               b0->current_data = l3_offset0;
1347               b0->current_length = mb->data_len - l3_offset0;
1348
1349               b0->flags = buffer_flags_template;
1350
1351               if (VMWARE_LENGTH_BUG_WORKAROUND)
1352                   b0->current_length -= 4;
1353
1354               vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
1355               vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
1356               vnet_buffer(b0)->io_handoff.next_index = next0;
1357               n_rx_bytes += mb->pkt_len;
1358
1359               /* Process subsequent segments of multi-segment packets */
1360               while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
1361                 {
1362                   ASSERT(mb_seg != 0);
1363
1364                   b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
1365                   vlib_buffer_init_for_free_list (b_seg, fl);
1366                   b_seg->clone_count = 0;
1367
1368                   ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1369                   ASSERT(b_seg->current_data == 0);
1370
1371                   /*
1372                    * The driver (e.g. virtio) may not put the packet data at the start
1373                    * of the segment, so don't assume b_seg->current_data == 0 is correct.
1374                    */
1375                   b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data;
1376
1377                   b_seg->current_length = mb_seg->data_len;
1378                   b0->total_length_not_including_first_buffer +=
1379                     mb_seg->data_len;
1380
1381                   b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
1382                   b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
1383
1384                   b_chain = b_seg;
1385                   mb_seg = mb_seg->next;
1386                   nb_seg++;
1387                 }
1388
1389               /*
1390                * Turn this on if you run into
1391                * "bad monkey" contexts, and you want to know exactly
1392                * which nodes they've visited... See main.c...
1393                */
1394               VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0);
1395
1396               if (PREDICT_FALSE (n_trace > mb_index))
1397                 vec_add1 (xd->d_trace_buffers, bi0);
1398
1399               next_worker_index = first_worker_index;
1400
1401               /*
1402                * Force unknown traffic onto worker 0,
1403                * and into ethernet-input. $$$$ add more hashes.
1404                */
1405               h0 = (ethernet_header_t *) b0->data;
1406
1407               /* Compute ingress LB hash */
1408               hash_key = eth_get_key(h0);
1409               hash = (u32)clib_xxhash(hash_key);
1410
1411               if (PREDICT_TRUE (is_pow2(num_workers)))
1412                 next_worker_index += hash & (num_workers - 1);
1413               else
1414                 next_worker_index += hash % num_workers;
1415
1416               /* if EFD is enabled and not already discarding from dpdk,
1417                * check the worker ring/queue for congestion
1418                */
1419               if (PREDICT_FALSE(tm->efd.enabled && !efd_discard_burst))
1420                 {
1421                   vlib_frame_queue_t *fq;
1422
1423                   /* fq will be valid if the ring is congested */
1424                   fq = is_vlib_handoff_queue_congested(
1425                       next_worker_index, tm->efd.queue_hi_thresh,
1426                       congested_handoff_queue_by_worker_index);
1427
1428                   if (PREDICT_FALSE(fq != NULL))
1429                     {
1430                       u32 cntr_type;
1431                       if (PREDICT_TRUE(cntr_type =
1432                                        is_efd_discardable(tm, b0, mb)))
1433                         {
1434                           /* discard the packet */
1435                           fq->enqueue_efd_discards++;
1436                           increment_efd_drop_counter(vm, cntr_type, 1);
1437                           rte_pktmbuf_free(mb);
1438                           n_buffers--;
1439                           mb_index++;
1440                           continue;
1441                         }
1442                     }
1443                 }
1444
1445               if (next_worker_index != current_worker_index)
1446                 {
1447                   if (hf)
1448                     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1449
1450                   hf = dpdk_get_handoff_queue_elt(
1451                            next_worker_index,
1452                            handoff_queue_elt_by_worker_index);
1453
1454                   n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1455                   to_next_worker = &hf->buffer_index[hf->n_vectors];
1456                   current_worker_index = next_worker_index;
1457                 }
1458
1459               /* enqueue to correct worker thread */
1460               to_next_worker[0] = bi0;
1461               to_next_worker++;
1462               n_left_to_next_worker--;
1463
1464               if (n_left_to_next_worker == 0)
1465                 {
1466                   hf->n_vectors = VLIB_FRAME_SIZE;
1467                   vlib_put_handoff_queue_elt(hf);
1468                   current_worker_index = ~0;
1469                   handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1470                   hf = 0;
1471                 }
1472
1473               n_buffers--;
1474               mb_index++;
1475             }
1476
1477           if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0))
1478             {
1479               /* credit the trace to the trace node */
1480               dpdk_rx_trace (dm, node_trace, xd, queue_id, xd->d_trace_buffers,
1481                              vec_len (xd->d_trace_buffers));
1482               vlib_set_trace_count (vm, node_trace, n_trace - vec_len (xd->d_trace_buffers));
1483             }
1484
1485           vlib_increment_combined_counter
1486             (vnet_get_main()->interface_main.combined_sw_if_counters
1487              + VNET_INTERFACE_COUNTER_RX,
1488              cpu_index,
1489              xd->vlib_sw_if_index,
1490              mb_index, n_rx_bytes);
1491
1492           dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index);
1493           dw->aggregate_rx_packets += mb_index;
1494         }
1495
1496       if (hf)
1497         hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1498
1499       /* Ship frames to the worker nodes */
1500       for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1501         {
1502           if (handoff_queue_elt_by_worker_index[i])
1503             {
1504               hf = handoff_queue_elt_by_worker_index[i];
1505               /*
1506                * It works better to let the handoff node
1507                * rate-adapt, always ship the handoff queue element.
1508                */
1509               if (1 || hf->n_vectors == hf->last_n_vectors)
1510                 {
1511                   vlib_put_handoff_queue_elt(hf);
1512                   handoff_queue_elt_by_worker_index[i] = 0;
1513                 }
1514               else
1515                 hf->last_n_vectors = hf->n_vectors;
1516             }
1517           congested_handoff_queue_by_worker_index[i] = (vlib_frame_queue_t *)(~0);
1518         }
1519       hf = 0;
1520       current_worker_index = ~0;
1521
1522       vlib_increment_main_loop_counter (vm);
1523     }
1524 }
1525
1526 /*
1527  * This function is used when the main thread performs IO and feeds the
1528  * worker threads.
1529  */
1530 static uword
1531 dpdk_io_input (vlib_main_t * vm,
1532                vlib_node_runtime_t * node,
1533                vlib_frame_t * f)
1534 {
1535   dpdk_main_t * dm = &dpdk_main;
1536   dpdk_device_t * xd;
1537   vlib_thread_main_t * tm = vlib_get_thread_main();
1538   uword n_rx_packets = 0;
1539   static vlib_frame_queue_elt_t ** handoff_queue_elt_by_worker_index;
1540   static vlib_frame_queue_t ** congested_handoff_queue_by_worker_index = 0;
1541   vlib_frame_queue_elt_t * hf = 0;
1542   int i;
1543   u32 n_left_to_next_worker = 0, * to_next_worker = 0;
1544   u32 next_worker_index = 0;
1545   u32 current_worker_index = ~0;
1546   u32 cpu_index = os_get_cpu_number();
1547   static int num_workers_set;
1548   static u32 num_workers;
1549   u16 queue_id = 0;
1550   vlib_node_runtime_t * node_trace;
1551   static u32 first_worker_index;
1552   u32 buffer_flags_template;
1553
1554   if (PREDICT_FALSE(num_workers_set == 0))
1555     {
1556       uword * p;
1557       vlib_thread_registration_t * tr;
1558       /* Only the standard vnet worker threads are supported */
1559       p = hash_get_mem (tm->thread_registrations_by_name, "workers");
1560       tr = (vlib_thread_registration_t *) p[0];
1561       if (tr)
1562         {
1563           num_workers = tr->count;
1564           first_worker_index = tr->first_index;
1565         }
1566       num_workers_set = 1;
1567     }
1568
1569   if (PREDICT_FALSE(handoff_queue_elt_by_worker_index == 0))
1570     {
1571       vec_validate (handoff_queue_elt_by_worker_index, tm->n_vlib_mains - 1);
1572
1573       vec_validate_init_empty (congested_handoff_queue_by_worker_index,
1574                                first_worker_index + num_workers - 1,
1575                                (vlib_frame_queue_t *)(~0));
1576     }
1577
1578   /* packet tracing is triggered on the dpdk-input node for ease-of-use */
1579   node_trace = vlib_node_get_runtime (vm, dpdk_input_node.index);
1580
1581   buffer_flags_template = dm->buffer_flags_template;
1582
1583   vec_foreach (xd, dm->devices)
1584     {
1585       u32 n_buffers;
1586       u32 mb_index;
1587       uword n_rx_bytes = 0;
1588       u32 n_trace, trace_cnt __attribute__((unused));
1589       vlib_buffer_free_list_t * fl;
1590       u32 hash;
1591       u64 hash_key;
1592       u8 efd_discard_burst = 0;
1593
1594       if (!xd->admin_up)
1595         continue;
1596
1597       n_buffers = dpdk_rx_burst(dm, xd, queue_id );
1598
1599       if (n_buffers == 0)
1600         {
1601           /* check if EFD (dpdk) is enabled */
1602           if (PREDICT_FALSE(dm->efd.enabled))
1603             {
1604               /* reset a few stats */
1605               xd->efd_agent.last_poll_time = 0;
1606               xd->efd_agent.last_burst_sz = 0;
1607             }
1608           continue;
1609         }
1610
1611       vec_reset_length (xd->d_trace_buffers);
1612       trace_cnt = n_trace = vlib_get_trace_count (vm, node_trace);
1613
1614       /*
1615        * DAW-FIXME: VMXNET3 device stop/start doesn't work,
1616        * therefore fake the stop in the dpdk driver by
1617        * silently dropping all of the incoming pkts instead of
1618        * stopping the driver / hardware.
1619        */
1620       if (PREDICT_FALSE(xd->admin_up != 1))
1621         {
1622           for (mb_index = 0; mb_index < n_buffers; mb_index++)
1623             rte_pktmbuf_free (xd->rx_vectors[queue_id][mb_index]);
1624           continue;
1625         }
1626
1627       /* Check for congestion if EFD (Early-Fast-Discard) is enabled
1628        * in any mode (e.g. dpdk, monitor, or drop_all)
1629        */
1630       if (PREDICT_FALSE(dm->efd.enabled))
1631         {
1632           /* update EFD counters */
1633           dpdk_efd_update_counters(xd, n_buffers, dm->efd.enabled);
1634
1635           if (PREDICT_FALSE(dm->efd.enabled & DPDK_EFD_DROPALL_ENABLED))
1636             {
1637               /* discard all received packets */
1638               for (mb_index = 0; mb_index < n_buffers; mb_index++)
1639                 rte_pktmbuf_free(xd->rx_vectors[queue_id][mb_index]);
1640
1641               xd->efd_agent.discard_cnt += n_buffers;
1642               increment_efd_drop_counter(vm,
1643                                          DPDK_ERROR_VLAN_EFD_DROP_PKTS,
1644                                          n_buffers);
1645
1646               continue;
1647             }
1648
1649           if (PREDICT_FALSE(xd->efd_agent.consec_full_frames_cnt >=
1650                             dm->efd.consec_full_frames_hi_thresh))
1651             {
1652               u32 device_queue_sz = rte_eth_rx_queue_count(xd->device_index,
1653                                                            queue_id);
1654               if (device_queue_sz >= dm->efd.queue_hi_thresh)
1655                 {
1656                   /* dpdk device queue has reached the critical threshold */
1657                   xd->efd_agent.congestion_cnt++;
1658
1659                   /* apply EFD to packets from the burst */
1660                   efd_discard_burst = 1;
1661                 }
1662             }
1663         }
1664
1665       fl = vlib_buffer_get_free_list
1666         (vm, VLIB_BUFFER_DEFAULT_FREE_LIST_INDEX);
1667
1668       mb_index = 0;
1669
1670       while (n_buffers > 0)
1671         {
1672           u32 bi0;
1673           u8 next0, error0;
1674           u32 l3_offset0;
1675           vlib_buffer_t * b0, * b_seg, * b_chain = 0;
1676           ethernet_header_t * h0;
1677           u8 nb_seg = 1;
1678           struct rte_mbuf *mb = xd->rx_vectors[queue_id][mb_index];
1679           struct rte_mbuf *mb_seg = mb->next;
1680
1681           if (PREDICT_TRUE(n_buffers > 1))
1682             {
1683               struct rte_mbuf *pfmb = xd->rx_vectors[queue_id][mb_index+2];
1684               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1685               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1686               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1687               CLIB_PREFETCH (bp->data, CLIB_CACHE_LINE_BYTES, LOAD);
1688             }
1689
1690           b0 = vlib_buffer_from_rte_mbuf(mb);
1691
1692           /* check whether EFD is looking for packets to discard */
1693           if (PREDICT_FALSE(efd_discard_burst))
1694             {
1695               u32 cntr_type;
1696               if (PREDICT_TRUE(cntr_type = is_efd_discardable(tm, b0, mb)))
1697                 {
1698                   rte_pktmbuf_free(mb);
1699                   xd->efd_agent.discard_cnt++;
1700                   increment_efd_drop_counter(vm,
1701                                              cntr_type,
1702                                              1);
1703
1704                   n_buffers--;
1705                   mb_index++;
1706                   continue;
1707                 }
1708             }
1709
1710           /* Prefetch one next segment if it exists */
1711           if (PREDICT_FALSE(mb->nb_segs > 1))
1712             {
1713               struct rte_mbuf *pfmb = mb->next;
1714               vlib_buffer_t *bp = vlib_buffer_from_rte_mbuf(pfmb);
1715               CLIB_PREFETCH (pfmb, CLIB_CACHE_LINE_BYTES, LOAD);
1716               CLIB_PREFETCH (bp, CLIB_CACHE_LINE_BYTES, STORE);
1717               b_chain = b0;
1718             }
1719
1720           bi0 = vlib_get_buffer_index (vm, b0);
1721           vlib_buffer_init_for_free_list (b0, fl);
1722           b0->clone_count = 0;
1723
1724           dpdk_rx_next_and_error_from_mb_flags_x1 (xd, mb, b0,
1725                                                    &next0, &error0);
1726 #ifdef RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS
1727           /*
1728            * Clear overloaded TX offload flags when a DPDK driver
1729            * is using them for RX flags (e.g. Cisco VIC Ethernet driver)
1730            */
1731           if (PREDICT_TRUE(trace_cnt == 0))
1732             mb->ol_flags &= PKT_EXT_RX_CLR_TX_FLAGS_MASK;
1733           else
1734             trace_cnt--;
1735 #endif /* RTE_LIBRTE_MBUF_EXT_RX_OLFLAGS */
1736
1737           if (error0)
1738             clib_warning ("bi %d error %d", bi0, error0);
1739
1740           b0->error = 0;
1741
1742           l3_offset0 = ((next0 == DPDK_RX_NEXT_IP4_INPUT ||
1743                          next0 == DPDK_RX_NEXT_IP6_INPUT ||
1744                          next0 == DPDK_RX_NEXT_MPLS_INPUT) ?
1745                         sizeof (ethernet_header_t) : 0);
1746
1747           b0->current_data = l3_offset0;
1748           b0->current_length = mb->data_len - l3_offset0;
1749
1750           b0->flags = buffer_flags_template;
1751
1752           if (VMWARE_LENGTH_BUG_WORKAROUND)
1753               b0->current_length -= 4;
1754
1755           vnet_buffer(b0)->sw_if_index[VLIB_RX] = xd->vlib_sw_if_index;
1756           vnet_buffer(b0)->sw_if_index[VLIB_TX] = (u32)~0;
1757           vnet_buffer(b0)->io_handoff.next_index = next0;
1758           n_rx_bytes += mb->pkt_len;
1759
1760           /* Process subsequent segments of multi-segment packets */
1761           while ((mb->nb_segs > 1) && (nb_seg < mb->nb_segs))
1762             {
1763               ASSERT(mb_seg != 0);
1764
1765               b_seg = vlib_buffer_from_rte_mbuf(mb_seg);
1766               vlib_buffer_init_for_free_list (b_seg, fl);
1767               b_seg->clone_count = 0;
1768
1769               ASSERT((b_seg->flags & VLIB_BUFFER_NEXT_PRESENT) == 0);
1770               ASSERT(b_seg->current_data == 0);
1771
1772               /*
1773                * The driver (e.g. virtio) may not put the packet data at the start
1774                * of the segment, so don't assume b_seg->current_data == 0 is correct.
1775                */
1776               b_seg->current_data = (mb_seg->buf_addr + mb_seg->data_off) - (void *)b_seg->data;
1777
1778               b_seg->current_length = mb_seg->data_len;
1779               b0->total_length_not_including_first_buffer +=
1780                 mb_seg->data_len;
1781
1782               b_chain->flags |= VLIB_BUFFER_NEXT_PRESENT;
1783               b_chain->next_buffer = vlib_get_buffer_index (vm, b_seg);
1784
1785               b_chain = b_seg;
1786               mb_seg = mb_seg->next;
1787               nb_seg++;
1788             }
1789
1790           /*
1791            * Turn this on if you run into
1792            * "bad monkey" contexts, and you want to know exactly
1793            * which nodes they've visited... See main.c...
1794            */
1795           VLIB_BUFFER_TRACE_TRAJECTORY_INIT(b0);
1796
1797           if (PREDICT_FALSE (n_trace > mb_index))
1798             vec_add1 (xd->d_trace_buffers, bi0);
1799
1800           next_worker_index = first_worker_index;
1801
1802           /*
1803            * Force unknown traffic onto worker 0,
1804            * and into ethernet-input. $$$$ add more hashes.
1805            */
1806           h0 = (ethernet_header_t *) b0->data;
1807
1808           /* Compute ingress LB hash */
1809           hash_key = eth_get_key(h0);
1810           hash = (u32)clib_xxhash(hash_key);
1811
1812           if (PREDICT_TRUE (is_pow2(num_workers)))
1813             next_worker_index += hash & (num_workers - 1);
1814           else
1815             next_worker_index += hash % num_workers;
1816
1817           /* if EFD is enabled and not already discarding from dpdk,
1818            * check the worker ring/queue for congestion
1819            */
1820           if (PREDICT_FALSE(tm->efd.enabled && !efd_discard_burst))
1821             {
1822               vlib_frame_queue_t *fq;
1823
1824               /* fq will be valid if the ring is congested */
1825               fq = is_vlib_handoff_queue_congested(
1826                   next_worker_index, tm->efd.queue_hi_thresh,
1827                   congested_handoff_queue_by_worker_index);
1828
1829               if (PREDICT_FALSE(fq != NULL))
1830                 {
1831                   u32 cntr_type;
1832                   if (PREDICT_TRUE(cntr_type =
1833                                    is_efd_discardable(tm, b0, mb)))
1834                     {
1835                       /* discard the packet */
1836                       fq->enqueue_efd_discards++;
1837                       increment_efd_drop_counter(vm, cntr_type, 1);
1838                       rte_pktmbuf_free(mb);
1839                       n_buffers--;
1840                       mb_index++;
1841                       continue;
1842                     }
1843                 }
1844             }
1845
1846           if (next_worker_index != current_worker_index)
1847             {
1848               if (hf)
1849                 hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1850
1851               hf = dpdk_get_handoff_queue_elt(
1852                      next_worker_index,
1853                      handoff_queue_elt_by_worker_index);
1854
1855               n_left_to_next_worker = VLIB_FRAME_SIZE - hf->n_vectors;
1856               to_next_worker = &hf->buffer_index[hf->n_vectors];
1857               current_worker_index = next_worker_index;
1858             }
1859
1860           /* enqueue to correct worker thread */
1861           to_next_worker[0] = bi0;
1862           to_next_worker++;
1863           n_left_to_next_worker--;
1864
1865           if (n_left_to_next_worker == 0)
1866             {
1867               hf->n_vectors = VLIB_FRAME_SIZE;
1868               vlib_put_handoff_queue_elt(hf);
1869               current_worker_index = ~0;
1870               handoff_queue_elt_by_worker_index[next_worker_index] = 0;
1871               hf = 0;
1872             }
1873
1874           n_buffers--;
1875           mb_index++;
1876         }
1877
1878       if (PREDICT_FALSE (vec_len (xd->d_trace_buffers) > 0))
1879         {
1880           /* credit the trace to the trace node */
1881           dpdk_rx_trace (dm, node_trace, xd, queue_id, xd->d_trace_buffers,
1882                          vec_len (xd->d_trace_buffers));
1883           vlib_set_trace_count (vm, node_trace, n_trace - vec_len (xd->d_trace_buffers));
1884         }
1885
1886       vlib_increment_combined_counter
1887         (vnet_get_main()->interface_main.combined_sw_if_counters
1888          + VNET_INTERFACE_COUNTER_RX,
1889          cpu_index,
1890          xd->vlib_sw_if_index,
1891          mb_index, n_rx_bytes);
1892
1893       dpdk_worker_t * dw = vec_elt_at_index(dm->workers, cpu_index);
1894       dw->aggregate_rx_packets += mb_index;
1895       n_rx_packets += mb_index;
1896     }
1897
1898   if (hf)
1899     hf->n_vectors = VLIB_FRAME_SIZE - n_left_to_next_worker;
1900
1901   /* Ship frames to the worker nodes */
1902   for (i = 0; i < vec_len (handoff_queue_elt_by_worker_index); i++)
1903     {
1904       if (handoff_queue_elt_by_worker_index[i])
1905         {
1906           hf = handoff_queue_elt_by_worker_index[i];
1907           /*
1908            * It works better to let the handoff node
1909            * rate-adapt, always ship the handoff queue element.
1910            */
1911           if (1 || hf->n_vectors == hf->last_n_vectors)
1912             {
1913               vlib_put_handoff_queue_elt(hf);
1914               handoff_queue_elt_by_worker_index[i] = 0;
1915             }
1916           else
1917             hf->last_n_vectors = hf->n_vectors;
1918         }
1919       congested_handoff_queue_by_worker_index[i] = (vlib_frame_queue_t *)(~0);
1920     }
1921   hf = 0;
1922   current_worker_index = ~0;
1923   return n_rx_packets;
1924 }
1925
1926 VLIB_REGISTER_NODE (dpdk_io_input_node) = {
1927   .function = dpdk_io_input,
1928   .type = VLIB_NODE_TYPE_INPUT,
1929   .name = "dpdk-io-input",
1930
1931   /* Will be enabled if/when hardware is detected. */
1932   .state = VLIB_NODE_STATE_DISABLED,
1933
1934   .format_buffer = format_ethernet_header_with_length,
1935   .format_trace = format_dpdk_rx_dma_trace,
1936
1937   .n_errors = DPDK_N_ERROR,
1938   .error_strings = dpdk_error_strings,
1939
1940   .n_next_nodes = DPDK_RX_N_NEXT,
1941   .next_nodes = {
1942     [DPDK_RX_NEXT_DROP] = "error-drop",
1943     [DPDK_RX_NEXT_ETHERNET_INPUT] = "ethernet-input",
1944     [DPDK_RX_NEXT_IP4_INPUT] = "ip4-input-no-checksum",
1945     [DPDK_RX_NEXT_IP6_INPUT] = "ip6-input",
1946     [DPDK_RX_NEXT_MPLS_INPUT] = "mpls-gre-input",
1947   },
1948 };
1949
1950 /*
1951  * set_efd_bitmap()
1952  * Based on the operation type, set lower/upper bits for the given index value
1953  */
1954 void
1955 set_efd_bitmap (u8 *bitmap, u32 value, u32 op)
1956 {
1957     int ix;
1958
1959     *bitmap = 0;
1960     for (ix = 0; ix < 8; ix++) {
1961         if (((op == EFD_OPERATION_LESS_THAN) && (ix < value)) ||
1962             ((op == EFD_OPERATION_GREATER_OR_EQUAL) && (ix >= value))){
1963             (*bitmap) |= (1 << ix);
1964         }
1965     }
1966 }
1967
1968 void
1969 efd_config (u32 enabled,
1970             u32 ip_prec,  u32 ip_op,
1971             u32 mpls_exp, u32 mpls_op,
1972             u32 vlan_cos, u32 vlan_op)
1973 {
1974    vlib_thread_main_t * tm = vlib_get_thread_main();
1975    dpdk_main_t * dm = &dpdk_main;
1976
1977    if (enabled) {
1978        tm->efd.enabled |= VLIB_EFD_DISCARD_ENABLED;
1979        dm->efd.enabled |= DPDK_EFD_DISCARD_ENABLED;
1980    } else {
1981        tm->efd.enabled &= ~VLIB_EFD_DISCARD_ENABLED;
1982        dm->efd.enabled &= ~DPDK_EFD_DISCARD_ENABLED;
1983    }
1984
1985    set_efd_bitmap(&tm->efd.ip_prec_bitmap, ip_prec, ip_op);
1986    set_efd_bitmap(&tm->efd.mpls_exp_bitmap, mpls_exp, mpls_op);
1987    set_efd_bitmap(&tm->efd.vlan_cos_bitmap, vlan_cos, vlan_op);
1988
1989 }