vxlan:offload RX flow
[vpp.git] / src / plugins / dpdk / device / node.c
index 3311ac4..602891d 100644 (file)
@@ -28,7 +28,7 @@
 
 #include <dpdk/device/dpdk_priv.h>
 
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
 static char *dpdk_error_strings[] = {
 #define _(n,s) s,
   foreach_dpdk_error
@@ -474,7 +474,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
   /* get up to DPDK_RX_BURST_SZ buffers from PMD */
   while (n_rx_packets < DPDK_RX_BURST_SZ)
     {
-      n = rte_eth_rx_burst (xd->device_index, queue_id,
+      n = rte_eth_rx_burst (xd->port_id, queue_id,
                            ptd->mbufs + n_rx_packets,
                            DPDK_RX_BURST_SZ - n_rx_packets);
       n_rx_packets += n;
@@ -500,7 +500,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
       next_index = xd->per_interface_next_index;
     }
 
-  /* as all packets belong to thr same interface feature arc lookup
+  /* as all packets belong to the same interface feature arc lookup
      can be don once and result stored in the buffer template */
   if (PREDICT_FALSE (vnet_device_input_have_features (xd->sw_if_index)))
     {
@@ -548,84 +548,12 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
   vlib_get_buffer_indices_with_offset (vm, (void **) ptd->mbufs, ptd->buffers,
                                       n_rx_packets,
                                       sizeof (struct rte_mbuf));
-  n_left = n_rx_packets;
-  next = ptd->next;
-  buffers = ptd->buffers;
-  mb = ptd->mbufs;
-  while (n_left)
-    {
-      u32 n_left_to_next;
-      u32 *to_next;
-      vlib_get_next_frame (vm, node, next_index, to_next, n_left_to_next);
-#ifdef CLIB_HAVE_VEC256
-      while (n_left >= 16 && n_left_to_next >= 16)
-       {
-         u16x16 next16 = u16x16_load_unaligned (next);
-         if (u16x16_is_all_equal (next16, next_index))
-           {
-             clib_memcpy (to_next, buffers, 16 * sizeof (u32));
-             to_next += 16;
-             n_left_to_next -= 16;
-             buffers += 16;
-             n_left -= 16;
-             next += 16;
-             mb += 16;
-           }
-         else
-           {
-             clib_memcpy (to_next, buffers, 4 * sizeof (u32));
-             to_next += 4;
-             n_left_to_next -= 4;
-
-             vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
-                                              n_left_to_next, buffers[0],
-                                              buffers[1], buffers[2],
-                                              buffers[3], next[0], next[1],
-                                              next[2], next[3]);
-             /* next */
-             buffers += 4;
-             n_left -= 4;
-             next += 4;
-             mb += 4;
-           }
-       }
-#endif
-      while (n_left >= 4 && n_left_to_next >= 4)
-       {
-         clib_memcpy (to_next, buffers, 4 * sizeof (u32));
-         to_next += 4;
-         n_left_to_next -= 4;
-
-         vlib_validate_buffer_enqueue_x4 (vm, node, next_index, to_next,
-                                          n_left_to_next, buffers[0],
-                                          buffers[1], buffers[2], buffers[3],
-                                          next[0], next[1], next[2],
-                                          next[3]);
-         /* next */
-         buffers += 4;
-         n_left -= 4;
-         next += 4;
-         mb += 4;
-       }
-      while (n_left && n_left_to_next)
-       {
-         clib_memcpy (to_next, buffers, 1 * sizeof (u32));
-         to_next += 1;
-         n_left_to_next -= 1;
-         vlib_validate_buffer_enqueue_x1 (vm, node, next_index, to_next,
-                                          n_left_to_next, buffers[0],
-                                          next[0]);
-         /* next */
-         buffers += 1;
-         n_left -= 1;
-         next += 1;
-         mb += 1;
-       }
-      vlib_put_next_frame (vm, node, next_index, n_left_to_next);
-    }
+
+  vlib_buffer_enqueue_to_next (vm, node, ptd->buffers, ptd->next,
+                              n_rx_packets);
 
   /* packet trace if enabled */
-  if ((n_trace = vlib_get_trace_count (vm, node)))
+  if (PREDICT_FALSE ((n_trace = vlib_get_trace_count (vm, node))))
     {
       n_left = n_rx_packets;
       buffers = ptd->buffers;
@@ -656,6 +584,48 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
       vlib_set_trace_count (vm, node, n_trace);
     }
 
+  /* rx pcap capture if enabled */
+  if (PREDICT_FALSE (dm->pcap[VLIB_RX].pcap_enable))
+    {
+      u32 bi0;
+      n_left = n_rx_packets;
+      buffers = ptd->buffers;
+      while (n_left)
+       {
+         bi0 = buffers[0];
+         b0 = vlib_get_buffer (vm, bi0);
+         buffers++;
+
+         if (dm->pcap[VLIB_RX].pcap_sw_if_index == 0 ||
+             dm->pcap[VLIB_RX].pcap_sw_if_index
+             == vnet_buffer (b0)->sw_if_index[VLIB_RX])
+           {
+             struct rte_mbuf *mb;
+             i16 data_start;
+             i32 temp_advance;
+
+             /*
+              * Note: current_data will have advanced
+              * when we skip ethernet input.
+              * Temporarily back up to the original DMA
+              * target, so we capture a valid ethernet frame
+              */
+             mb = rte_mbuf_from_vlib_buffer (b0);
+
+             /* Figure out the original data_start */
+             data_start = (mb->buf_addr + mb->data_off) - (void *) b0->data;
+             /* Back up that far */
+             temp_advance = b0->current_data - data_start;
+             vlib_buffer_advance (b0, -temp_advance);
+             /* Trace the packet */
+             pcap_add_buffer (&dm->pcap[VLIB_RX].pcap_main, vm, bi0, 512);
+             /* and advance again */
+             vlib_buffer_advance (b0, temp_advance);
+           }
+         n_left--;
+       }
+    }
+
   vlib_increment_combined_counter
     (vnet_get_main ()->interface_main.combined_sw_if_counters
      + VNET_INTERFACE_COUNTER_RX, thread_index, xd->sw_if_index,
@@ -666,8 +636,7 @@ dpdk_device_input (vlib_main_t * vm, dpdk_main_t * dm, dpdk_device_t * xd,
   return n_rx_packets;
 }
 
-uword CLIB_CPU_OPTIMIZED
-CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
+VLIB_NODE_FN (dpdk_input_node) (vlib_main_t * vm, vlib_node_runtime_t * node,
                                vlib_frame_t * f)
 {
   dpdk_main_t *dm = &dpdk_main;
@@ -696,10 +665,9 @@ CLIB_MULTIARCH_FN (dpdk_input) (vlib_main_t * vm, vlib_node_runtime_t * node,
   return n_rx_packets;
 }
 
-#ifndef CLIB_MULTIARCH_VARIANT
+#ifndef CLIB_MARCH_VARIANT
 /* *INDENT-OFF* */
 VLIB_REGISTER_NODE (dpdk_input_node) = {
-  .function = dpdk_input,
   .type = VLIB_NODE_TYPE_INPUT,
   .name = "dpdk-input",
   .sibling_of = "device-input",
@@ -714,20 +682,6 @@ VLIB_REGISTER_NODE (dpdk_input_node) = {
   .error_strings = dpdk_error_strings,
 };
 /* *INDENT-ON* */
-
-vlib_node_function_t __clib_weak dpdk_input_avx512;
-vlib_node_function_t __clib_weak dpdk_input_avx2;
-
-#if __x86_64__
-static void __clib_constructor
-dpdk_input_multiarch_select (void)
-{
-  if (dpdk_input_avx512 && clib_cpu_supports_avx512f ())
-    dpdk_input_node.function = dpdk_input_avx512;
-  else if (dpdk_input_avx2 && clib_cpu_supports_avx2 ())
-    dpdk_input_node.function = dpdk_input_avx2;
-}
-#endif
 #endif
 
 /*